<commit_before>module HaskellTools.Hackage where

import qualified Distribution.Hackage.DB as DB
import Distribution.PackageDescription

import Control.Monad (unless)
import Pipes

haskellPackages :: Int -> Producer [PackageDescription] IO ()
haskellPackages page = do
  pkgs <- lift $ packages page
  unless (null pkgs) $ do
    yield pkgs
    haskellPackages $ page + 1

packages :: Int -> IO [PackageDescription]
packages page = do
  db <- DB.readHackage
  return $ pageSlice $ pkgDescriptions $ DB.toAscList db
    where
      latestVersion = head . DB.toDescList . snd
      pkgDescriptions = map (packageDescription . snd . latestVersion)
      slice from to xs = take (to - from + 1) (drop from xs)
      pageSlice = slice (page * pageSize) (page * pageSize + pageSize)
      pageSize = 1000<commit_msg>Simplify code<commit_after>module HaskellTools.Hackage where

import qualified Distribution.Hackage.DB as DB
import Distribution.PackageDescription

import Control.Monad (unless, liftM)
import Pipes

haskellPackages :: Int -> Producer [PackageDescription] IO ()
haskellPackages page = do
  pkgs <- lift $ packages page
  unless (null pkgs) $ do
    yield pkgs
    haskellPackages $ page + 1

packages :: Int -> IO [PackageDescription]
packages page = liftM (pageSlice . pkgDescriptions . DB.toAscList) DB.readHackage
    where
      latestVersion = head . DB.toDescList . snd
      pkgDescriptions = map (packageDescription . snd . latestVersion)
      slice from to xs = take (to - from + 1) (drop from xs)
      pageSlice = slice (page * pageSize) (page * pageSize + pageSize)
      pageSize = 1000
{'fraction_non_alphanumeric': 0.08198757763975155, 'fraction_numerical': 0.007453416149068323, 'mean_word_length': 3.5380281690140847, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1654868', 'n_tokens_mistral': 506, 'n_tokens_neox': 454, 'n_words': 158}