<commit_before>{-# LANGUAGE DefaultSignatures,
             OverloadedStrings,
             TupleSections,
             LambdaCase,
             TemplateHaskell #-}
{-
  GHCJS linker, collects dependencies from
    the object files (.js_o), which contain linkable 
    units with dependency information
-}
module Gen2.Linker where

import           DynFlags
import           Module                   (PackageId, packageIdString)

import           Control.Applicative
import           Control.Concurrent.MVar
import           Control.Monad
import           Control.Parallel.Strategies

import           Data.Array
import qualified Data.ByteString.Lazy     as BL
import           Data.Function            (on)
import           Data.Int
import qualified Data.IntSet              as IS
import           Data.List                ( partition, isPrefixOf, nub
                                          , intercalate, group, sort, groupBy)
import           Data.Map.Strict          (Map)
import qualified Data.Map.Strict          as M
import           Data.Maybe               (fromMaybe, isJust, isNothing)
import           Data.Monoid
import           Data.Set                 (Set)
import qualified Data.Set                 as S
import           Data.Text                (Text)
import qualified Data.Text                as T
import qualified Data.Text.IO             as T
import qualified Data.Text.Lazy           as TL
import qualified Data.Text.Lazy.IO        as TL
import qualified Data.Text.Lazy.Encoding  as TLE

import           System.FilePath          (splitPath, (<.>), (</>))
import           System.Directory         (createDirectoryIfMissing, doesFileExist)

import           Text.PrettyPrint.Leijen.Text (displayT, renderPretty)

import           Compiler.Info
import           Compiler.JMacro
import           Compiler.Settings

import           Gen2.ClosureInfo         hiding (Fun)
import qualified Gen2.Compactor           as Compactor
import           Gen2.Object
import           Gen2.Printer             (pretty)
import           Gen2.Utils
import           Gen2.Rts                 (rtsText)
import           Gen2.RtsTypes
import           Gen2.Shim


type LinkableUnit = (Package, Module, Int) -- module and the index of the block
type Module       = Text

-- number of bytes linked per module
type LinkerStats  = Map (Package, Module) Int64

link :: DynFlags
     -> GhcjsSettings
     -> FilePath                  -- ^ output file/directory
     -> [FilePath]                -- ^ include path for home package
     -> [(PackageId, [FilePath])] -- ^ directories to load package modules
     -> [FilePath]                -- ^ the object files we're linking
     -> [FilePath]                -- ^ extra js files to include
     -> (Fun -> Bool)             -- ^ functions from the objects to use as roots (include all their deps)
     -> IO ()
link dflags settings out include pkgs objFiles jsFiles isRootFun
  | gsNativeExecutables settings = return ()
  | otherwise = do
  objDeps <- mapM readDepsFile objFiles
  let genBase = isJust (gsGenBase settings)
      jsExt | genBase   = "base.js"
            | otherwise = "js"
      debug = buildingDebug dflags
      rootTest | Just baseMod <- gsGenBase settings =
                   \(Fun p m s) -> m == T.pack baseMod
               | otherwise = isRootFun
      roots = S.fromList . filter rootTest $
        concatMap (map fst . M.toList . depsDeps) objDeps
      rootMods = map (T.unpack . head) . group . sort . map funModule . S.toList $ roots
  -- putStrLn ("objects: " ++ show (traverse . _1 %~ packageIdString $ pkgs))
  compilationProgressMsg dflags $
    case gsGenBase settings of
      Just baseMod -> "Linking base bundle " ++ out ++ " (" ++ baseMod ++ ")"
      _            -> "Linking " ++ out ++ " (" ++ intercalate "," rootMods ++ ")"
  base <- Compactor.loadBase (gsUseBase settings)
  c <- newMVar M.empty
  (allDeps, code) <-
    collectDeps (lookupFun c $ zip objDeps objFiles)
                (Compactor.baseUnits base)
                (roots `S.union` rtsDeps (map fst pkgs))
  createDirectoryIfMissing False out
  let (outJs, metaSize, compactorState, stats) = renderLinker settings dflags (Compactor.baseCompactorState base) code
      pkgs' = filter (\p -> T.pack (packageIdString p) `notElem` Compactor.basePkgs base) (map fst pkgs)
      pkgsT = map (T.pack . packageIdString) pkgs'
  BL.writeFile (out </> "out" <.> jsExt) outJs
  when (not $ gsOnlyOut settings) $ do
    when (not $ gsNoStats settings) $ do
      let statsFile = if genBase then "out.base.stats" else "out.stats"<commit_msg>Fix --native-executables and --no-js-executables<commit_after>{-# LANGUAGE DefaultSignatures,
             OverloadedStrings,
             TupleSections,
             LambdaCase,
             TemplateHaskell #-}
{-
  GHCJS linker, collects dependencies from
    the object files (.js_o), which contain linkable 
    units with dependency information
-}
module Gen2.Linker where

import           DynFlags
import           Module                   (PackageId, packageIdString)

import           Control.Applicative
import           Control.Concurrent.MVar
import           Control.Monad
import           Control.Parallel.Strategies

import           Data.Array
import qualified Data.ByteString.Lazy     as BL
import           Data.Function            (on)
import           Data.Int
import qualified Data.IntSet              as IS
import           Data.List                ( partition, isPrefixOf, nub
                                          , intercalate, group, sort, groupBy)
import           Data.Map.Strict          (Map)
import qualified Data.Map.Strict          as M
import           Data.Maybe               (fromMaybe, isJust, isNothing)
import           Data.Monoid
import           Data.Set                 (Set)
import qualified Data.Set                 as S
import           Data.Text                (Text)
import qualified Data.Text                as T
import qualified Data.Text.IO             as T
import qualified Data.Text.Lazy           as TL
import qualified Data.Text.Lazy.IO        as TL
import qualified Data.Text.Lazy.Encoding  as TLE

import           System.FilePath          (splitPath, (<.>), (</>))
import           System.Directory         (createDirectoryIfMissing, doesFileExist)

import           Text.PrettyPrint.Leijen.Text (displayT, renderPretty)

import           Compiler.Info
import           Compiler.JMacro
import           Compiler.Settings

import           Gen2.ClosureInfo         hiding (Fun)
import qualified Gen2.Compactor           as Compactor
import           Gen2.Object
import           Gen2.Printer             (pretty)
import           Gen2.Utils
import           Gen2.Rts                 (rtsText)
import           Gen2.RtsTypes
import           Gen2.Shim


type LinkableUnit = (Package, Module, Int) -- module and the index of the block
type Module       = Text

-- number of bytes linked per module
type LinkerStats  = Map (Package, Module) Int64

link :: DynFlags
     -> GhcjsSettings
     -> FilePath                  -- ^ output file/directory
     -> [FilePath]                -- ^ include path for home package
     -> [(PackageId, [FilePath])] -- ^ directories to load package modules
     -> [FilePath]                -- ^ the object files we're linking
     -> [FilePath]                -- ^ extra js files to include
     -> (Fun -> Bool)             -- ^ functions from the objects to use as roots (include all their deps)
     -> IO ()
link dflags settings out include pkgs objFiles jsFiles isRootFun
  | gsNoJSExecutables settings = return ()
  | otherwise = do
  objDeps <- mapM readDepsFile objFiles
  let genBase = isJust (gsGenBase settings)
      jsExt | genBase   = "base.js"
            | otherwise = "js"
      debug = buildingDebug dflags
      rootTest | Just baseMod <- gsGenBase settings =
                   \(Fun p m s) -> m == T.pack baseMod
               | otherwise = isRootFun
      roots = S.fromList . filter rootTest $
        concatMap (map fst . M.toList . depsDeps) objDeps
      rootMods = map (T.unpack . head) . group . sort . map funModule . S.toList $ roots
  -- putStrLn ("objects: " ++ show (traverse . _1 %~ packageIdString $ pkgs))
  compilationProgressMsg dflags $
    case gsGenBase settings of
      Just baseMod -> "Linking base bundle " ++ out ++ " (" ++ baseMod ++ ")"
      _            -> "Linking " ++ out ++ " (" ++ intercalate "," rootMods ++ ")"
  base <- Compactor.loadBase (gsUseBase settings)
  c <- newMVar M.empty
  (allDeps, code) <-
    collectDeps (lookupFun c $ zip objDeps objFiles)
                (Compactor.baseUnits base)
                (roots `S.union` rtsDeps (map fst pkgs))
  createDirectoryIfMissing False out
  let (outJs, metaSize, compactorState, stats) = renderLinker settings dflags (Compactor.baseCompactorState base) code
      pkgs' = filter (\p -> T.pack (packageIdString p) `notElem` Compactor.basePkgs base) (map fst pkgs)
      pkgsT = map (T.pack . packageIdString) pkgs'
  BL.writeFile (out </> "out" <.> jsExt) outJs
  when (not $ gsOnlyOut settings) $ do
    when (not $ gsNoStats settings) $ do
      let statsFile = if genBase then "out.base.stats" else "out.stats"
{'fraction_non_alphanumeric': 0.08480833966771636, 'fraction_numerical': 0.0026061461613638833, 'mean_word_length': 1.8826291079812207, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7276230', 'n_tokens_mistral': 2575, 'n_tokens_neox': 2490, 'n_words': 794}