<commit_before>{-# LANGUAGE NoImplicitPrelude, OverloadedStrings, NoMonomorphismRestriction, ExtendedDefaultRules #-}
{-
   Boot program to build the GHCJS core libraries from a configured GHC source tree

   steps:
   1. download ghc sources and extract to temp directory (only if autobooting)
   2. make stage1 compiler and tools (only if autobooting)
   3. generate inplace/bin/ghcjs and inplace/bin/ghcjs-pkg wrappers
   4. build libraries with ghcjs
   5. install libraries with ghc-cabal
   6. copy .js files for libraries to installation locations
-}

module Main where

import Prelude hiding (FilePath)
import qualified Data.Text.Lazy as T
import Data.Text.Lazy (Text)
import Shelly
import Data.Monoid (mappend)
import Control.Monad (forM, forM_)
import Compiler.Info
import qualified Data.ByteString.Lazy as L
import Filesystem.Path.CurrentOS (stripPrefix, empty)
import Data.Maybe (catMaybes)

import System.Directory
import System.Environment
import Distribution.Simple.Utils (withTempDirectory)
import Distribution.Verbosity (normal)
import Network (withSocketsDo)

import qualified Codec.Archive.Tar as Tar
import qualified Codec.Archive.Tar.Check as Tar

import Data.Conduit (($$), ($=), (=$=))
import Data.Conduit.Lazy (lazyConsume)
import Data.Conduit.BZlib
import Network.HTTP.Conduit
import System.Directory ( setPermissions, getPermissions, Permissions(..) )

default (T.Text)

(<>) = mappend

ghcDownloadLocation :: String -> String
ghcDownloadLocation ver =
--    "http://server/ghc/dist/" ++ ver ++ "/ghc-" ++ ver ++ "-src.tar.bz2"
    "http://www.haskell.org/ghc/dist/" ++ ver ++ "/ghc-" ++ ver ++ "-src.tar.bz2"

main = withSocketsDo $ do
  args <- getArgs
  tmp <- getTemporaryDirectory
  withTempDirectory normal tmp "ghcjs-boot" $ \tmpDir -> shelly $ do
    when (any (=="--auto") args) (autoBoot $ fromString tmpDir)
    installBootPackages

fromString :: String -> FilePath
fromString = fromText . T.pack

toString :: FilePath -> String
toString = T.unpack . toTextIgnore

-- autoboots roll out
-- download and configure a fresh ghc source tree
autoBoot :: FilePath -> ShIO ()
autoBoot tmp = shelly $ do
  ghcVer <- T.unpack . T.strip <$> run "ghc" ["--numeric-version"]
  let ghcDir = "ghc-" ++ ghcVer
  cd tmp
  liftIO $ do
    putStrLn $ "fetching GHC: " ++ ghcDownloadLocation ghcVer
    request <- parseUrl (ghcDownloadLocation ghcVer)
    withManager $ \manager -> do
                          Response _ _ _ src <- http request manager
                          tar <- lazyConsume (src =$= bunzip2)
                          liftIO $ putStrLn "unpacking tar"
                          liftIO $ Tar.unpack (toString tmp)  (Tar.read $ L.fromChunks tar) -- (Tar.checkTarbomb ("ghc-" ++ ghcVer) $ Tar.read tar)
  cd (tmp </> T.pack ghcDir)
  run_ "sh" ["configure"]
  run_ "make" ["stage1"]

-- to be called from a configured and built (at least stage 1) ghc tree
installBootPackages :: ShIO ()
installBootPackages = do
  mghcjs <- which "ghcjs"
  mghcjspkg <- which "ghcjs-pkg"
  case (mghcjs, mghcjspkg) of
    (Just ghcjs, Just ghcjspkg) -> do
      echo $ "Booting: " <> toTextIgnore ghcjs <> " (" <> toTextIgnore ghcjspkg <> ")"
      initPackageDB
      addWrappers ghcjs ghcjspkg
      run_ "make" ["all_rts", "-j4"] `catchany_sh` (const (return ()))
      run_ "make" ["all_libraries", "-j4", "GHC_STAGE1=inplace/bin/ghcjs", "GHC_PKG_PGM=ghcjs-pkg"] `catchany_sh` (const (return ()))
      installRts
      mapM_ (installPkg ghcjs ghcjspkg) ["ghc-prim", "integer-gmp", "base"]
    _ -> echo "Error: ghcjs and ghcjs-pkg must be in the PATH"
<commit_msg>some booter improvements<commit_after>{-# LANGUAGE NoImplicitPrelude, OverloadedStrings, NoMonomorphismRestriction, ExtendedDefaultRules #-}
{-
   Boot program to build the GHCJS core libraries from a configured GHC source tree

   steps:
   1. download ghc sources and extract to temp directory (only if autobooting)
   2. make stage1 compiler and tools (only if autobooting)
   3. generate inplace/bin/ghcjs and inplace/bin/ghcjs-pkg wrappers
   4. build libraries with ghcjs
   5. install libraries with ghc-cabal
   6. copy .js files for libraries to installation locations
-}

module Main where

import Prelude hiding (FilePath)
import qualified Data.Text.Lazy as T
import Data.Text.Lazy (Text)
import Shelly
import Data.Monoid (mappend)
import Control.Monad (forM, forM_)
import Compiler.Info
import qualified Data.ByteString.Lazy as L
import Filesystem.Path.CurrentOS (stripPrefix, empty)
import Data.Maybe (catMaybes)

import System.Directory
import System.Environment
import Distribution.Simple.Utils (withTempDirectory)
import Distribution.Verbosity (normal)
import Network (withSocketsDo)

import qualified Codec.Archive.Tar as Tar
import qualified Codec.Archive.Tar.Check as Tar

import Data.Conduit (($$), ($=), (=$=))
import Data.Conduit.Lazy (lazyConsume)
import Data.Conduit.BZlib
import Network.HTTP.Conduit
import System.Directory ( setPermissions, getPermissions, Permissions(..) )

default (T.Text)

(<>) = mappend

ghcDownloadLocation :: String -> String
ghcDownloadLocation ver =
--    "http://server/ghc/dist/" ++ ver ++ "/ghc-" ++ ver ++ "-src.tar.bz2"
    "http://www.haskell.org/ghc/dist/" ++ ver ++ "/ghc-" ++ ver ++ "-src.tar.bz2"

data BootSettings = BootSettings { skipRts :: Bool } deriving (Ord, Eq)

bootSettings :: [String] -> BootSettings
bootSettings args = BootSettings (any (=="--skip-rts") args)

main = withSocketsDo $ do
  args <- getArgs
  tmp <- getTemporaryDirectory
  withTempDirectory normal tmp "ghcjs-boot" $ \tmpDir -> shelly $ do
    when (any (=="--auto") args) (autoBoot $ fromString tmpDir)
    installBootPackages (bootSettings args)

fromString :: String -> FilePath
fromString = fromText . T.pack

toString :: FilePath -> String
toString = T.unpack . toTextIgnore

-- autoboots roll out
-- download and configure a fresh ghc source tree
autoBoot :: FilePath -> ShIO ()
autoBoot tmp = shelly $ do
  ghcVer <- T.unpack . T.strip <$> run "ghc" ["--numeric-version"]
  let ghcDir = "ghc-" ++ ghcVer
  cd tmp
  liftIO $ do
    putStrLn $ "fetching GHC: " ++ ghcDownloadLocation ghcVer
    request <- parseUrl (ghcDownloadLocation ghcVer)
    withManager $ \manager -> do
                          Response _ _ _ src <- http request manager
                          tar <- lazyConsume (src =$= bunzip2)
                          liftIO $ putStrLn "unpacking tar"
                          liftIO $ Tar.unpack (toString tmp)  (Tar.read $ L.fromChunks tar) -- (Tar.checkTarbomb ("ghc-" ++ ghcVer) $ Tar.read tar)
  cd (tmp </> T.pack ghcDir)
  run_ "sh" ["configure"]
  run_ "make" ["stage1"]

corePackages :: [Text]
corePackages = ["ghc-prim", "integer-gmp", "base"]

-- to be called from a configured and built (at least stage 1) ghc tree
installBootPackages :: BootSettings -> ShIO ()
installBootPackages settings = do
  mghcjs <- which "ghcjs"
  mghcjspkg <- which "ghcjs-pkg"
  case (mghcjs, mghcjspkg) of
    (Just ghcjs, Just ghcjspkg) -> do
      echo $ "Booting: " <> toTextIgnore ghcjs <> " (" <> toTextIgnore ghcjspkg <> ")"
      initPackageDB
      addWrappers ghcjs ghcjspkg
      when (not $ skipRts settings) $
           run_ "make" ["all_rts", "-j4"] `catchany_sh` (const (return ()))
      forM_ corePackages $ \pkg ->
        run_ "make" ["all_libraries/"<>pkg, "-j4", "GHC_STAGE1=inplace/bin/ghcjs", "GHC_PKG_PGM=ghcjs-pkg"] `catchany_sh` (const (return ()))
      installRts
      mapM_ (installPkg ghcjs ghcjspkg) corePackages
    _ -> echo "Error: ghcjs and ghcjs-pkg must be in the PATH"

{'fraction_non_alphanumeric': 0.12073073868149325, 'fraction_numerical': 0.003971405877680699, 'mean_word_length': 3.410391126678342, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6556761', 'n_tokens_mistral': 2553, 'n_tokens_neox': 2485, 'n_words': 734}