<commit_before>{-# LANGUAGE DeriveDataTypeable, GeneralizedNewtypeDeriving #-}

module Oracles.PackageData (
    PackageData (..), PackageDataList (..),
    pkgData, pkgDataList, packageDataOracle
    ) where

import Base
import Util
import Data.List
import Data.Maybe
import Control.Applicative
import qualified Data.HashMap.Strict as Map

-- For each (PackageData path) the file 'path/package-data.mk' contains
-- a line of the form 'path_VERSION = 1.2.3.4'.
-- pkgData $ PackageData path is an action that consults the file and
-- returns "1.2.3.4".
--
-- PackageDataList is used for multiple string options separated by spaces,
-- such as 'path_MODULES = Data.Array Data.Array.Base ...'.
-- pkgListData Modules therefore returns ["Data.Array", "Data.Array.Base", ...]
data PackageData = Version      FilePath
                 | PackageKey   FilePath
                 | Synopsis     FilePath
                 | BuildGhciLib FilePath

data PackageDataList = Modules        FilePath
                     | HiddenModules  FilePath
                     | SrcDirs        FilePath
                     | IncludeDirs    FilePath
                     | Deps           FilePath
                     | DepKeys        FilePath
                     | DepNames       FilePath
                     | CppArgs        FilePath
                     | HsArgs         FilePath
                     | CcArgs         FilePath
                     | CSrcs          FilePath
                     | DepIncludeDirs FilePath

newtype PackageDataKey = PackageDataKey (FilePath, String)
    deriving (Show, Typeable, Eq, Hashable, Binary, NFData)

-- TODO: is this needed?
askPackageData :: FilePath -> String -> Action String
askPackageData path key = do
    let fullKey = replaceSeparators '_' $ path ++ "_" ++ key
        pkgData = path -/- "package-data.mk"
    value <- askOracle $ PackageDataKey (pkgData, fullKey)
    return $ fromMaybe
        (error $ "No key '" ++ key ++ "' in " ++ pkgData ++ ".") value

pkgData :: PackageData -> Action String
pkgData packageData = do
    let (key, path) = case packageData of
           Version      path -> ("VERSION"       , path)
           PackageKey   path -> ("PACKAGE_KEY"   , path)
           Synopsis     path -> ("SYNOPSIS"      , path)
           BuildGhciLib path -> ("BUILD_GHCI_LIB", path)
        fullKey = replaceSeparators '_' $ path ++ "_" ++ key
        pkgData = path -/- "package-data.mk"
    res <- askOracle $ PackageDataKey (pkgData, fullKey)
    return $ fromMaybe
        (error $ "No key '" ++ key ++ "' in " ++ pkgData ++ ".") res

pkgDataList :: PackageDataList -> Action [String]
pkgDataList packageData = do
    let (key, path, defaultValue) = case packageData of
           Modules        path -> ("MODULES"                       , path, "" )
           HiddenModules  path -> ("HIDDEN_MODULES"                , path, "" )
           SrcDirs        path -> ("HS_SRC_DIRS"                   , path, ".")
           IncludeDirs    path -> ("INCLUDE_DIRS"                  , path, ".")
           Deps           path -> ("DEPS"                          , path, "" )
           DepKeys        path -> ("DEP_KEYS"                      , path, "" )
           DepNames       path -> ("DEP_NAMES"                     , path, "" )
           CppArgs        path -> ("CPP_OPTS"                      , path, "" )
           HsArgs         path -> ("HC_OPTS"                       , path, "" )
           CcArgs         path -> ("CC_OPTS"                       , path, "" )
           CSrcs          path -> ("C_SRCS"                        , path, "" )
           DepIncludeDirs path -> ("DEP_INCLUDE_DIRS_SINGLE_QUOTED", path, "" )
        fullKey = replaceSeparators '_' $ path ++ "_" ++ key
        pkgData = path -/- "package-data.mk"
        unquote = dropWhile (== '\'') . dropWhileEnd (== '\'')
    res <- askOracle $ PackageDataKey (pkgData, fullKey)
    return $ map unquote $ words $ case res of
        Nothing    -> error $ "No key '" ++ key ++ "' in " ++ pkgData ++ "."
        Just ""    -> defaultValue
        Just value -> value

-- Oracle for 'package-data.mk' files
packageDataOracle :: Rules ()
packageDataOracle = do
    pkgData <- newCache $ \file -> do
        need [file]
        putOracle $ "Reading " ++ file ++ "..."
        liftIO $ readConfigFile file
    addOracle $ \(PackageDataKey (file, key)) ->
        Map.lookup key <$> pkgData (unifyPath file)
    return ()<commit_msg>Drop DepKeys, add DepId, clean up code.<commit_after>{-# LANGUAGE DeriveDataTypeable, GeneralizedNewtypeDeriving #-}

module Oracles.PackageData (
    PackageData (..), PackageDataList (..),
    pkgData, pkgDataList, packageDataOracle
    ) where

import Base
import Util
import Data.List
import Control.Applicative
import qualified Data.HashMap.Strict as Map

-- For each (PackageData path) the file 'path/package-data.mk' contains
-- a line of the form 'path_VERSION = 1.2.3.4'.
-- pkgData $ PackageData path is an action that consults the file and
-- returns "1.2.3.4".
--
-- PackageDataList is used for multiple string options separated by spaces,
-- such as 'path_MODULES = Data.Array Data.Array.Base ...'.
-- pkgListData Modules therefore returns ["Data.Array", "Data.Array.Base", ...]
data PackageData = Version      FilePath
                 | PackageKey   FilePath
                 | LibName      FilePath
                 | Synopsis     FilePath
                 | BuildGhciLib FilePath

data PackageDataList = Modules        FilePath
                     | HiddenModules  FilePath
                     | SrcDirs        FilePath
                     | IncludeDirs    FilePath
                     | Deps           FilePath
                     | DepIds         FilePath
                     | DepNames       FilePath
                     | CppArgs        FilePath
                     | HsArgs         FilePath
                     | CcArgs         FilePath
                     | CSrcs          FilePath
                     | DepIncludeDirs FilePath

newtype PackageDataKey = PackageDataKey (FilePath, String)
    deriving (Show, Typeable, Eq, Hashable, Binary, NFData)

askPackageData :: FilePath -> String -> Action String
askPackageData path key = do
    let fullKey = replaceSeparators '_' $ path ++ "_" ++ key
        file    = path -/- "package-data.mk"
    maybeValue <- askOracle $ PackageDataKey (file, fullKey)
    case maybeValue of
        Nothing    -> putError $ "No key '" ++ key ++ "' in " ++ file ++ "."
        Just value -> return value

pkgData :: PackageData -> Action String
pkgData packageData = case packageData of
    Version      path -> askPackageData path "VERSION"
    PackageKey   path -> askPackageData path "PACKAGE_KEY"
    LibName      path -> askPackageData path "LIB_NAME"
    Synopsis     path -> askPackageData path "SYNOPSIS"
    BuildGhciLib path -> askPackageData path "BUILD_GHCI_LIB"

pkgDataList :: PackageDataList -> Action [String]
pkgDataList packageData = fmap (map unquote . words) $ case packageData of
    Modules        path -> askPackageData path "MODULES"
    HiddenModules  path -> askPackageData path "HIDDEN_MODULES"
    SrcDirs        path -> askPackageData path "HS_SRC_DIRS"
    IncludeDirs    path -> askPackageData path "INCLUDE_DIRS"
    Deps           path -> askPackageData path "DEPS"
    DepIds         path -> askPackageData path "DEP_IPIDS"
    DepNames       path -> askPackageData path "DEP_NAMES"
    CppArgs        path -> askPackageData path "CPP_OPTS"
    HsArgs         path -> askPackageData path "HC_OPTS"
    CcArgs         path -> askPackageData path "CC_OPTS"
    CSrcs          path -> askPackageData path "C_SRCS"
    DepIncludeDirs path -> askPackageData path "DEP_INCLUDE_DIRS_SINGLE_QUOTED"
  where
    unquote = dropWhile (== '\'') . dropWhileEnd (== '\'')

-- Oracle for 'package-data.mk' files
packageDataOracle :: Rules ()
packageDataOracle = do
    pkgDataContents <- newCache $ \file -> do
        need [file]
        putOracle $ "Reading " ++ file ++ "..."
        liftIO $ readConfigFile file
    _ <- addOracle $ \(PackageDataKey (file, key)) ->
        Map.lookup key <$> pkgDataContents file
    return ()
{'fraction_non_alphanumeric': 0.10252327290543851, 'fraction_numerical': 0.0019598236158745713, 'mean_word_length': 1.831137309292649, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4908280', 'n_tokens_mistral': 2319, 'n_tokens_neox': 2200, 'n_words': 641}