<commit_before>{-# LANGUAGE Rank2Types #-}

module Command.Dupes.Keep (
    Options
  , parserInfo
  , run
) where

import Command.ParseUtil
import Dupes
import Store.LevelDB

import Control.DeepSeq
import Control.Exception
import Control.Monad
import Control.Monad.Trans ( lift )
import Data.List ( delete, (\\), sort, isPrefixOf )
import Data.Set (Set)
import qualified Data.Set as Set
import Data.Machine hiding ( run )
import Data.Machine.Fanout
import Data.Machine.Interleave
import Options.Applicative
import qualified Data.ByteString.Lazy as L
import System.Directory
import System.FilePath ( (</>) )
import System.IO
import Debug.Trace

data Options = Options
  { optStdin :: Bool
  , optPaths :: [FilePath]  }

parserInfo :: ParserInfo Options
parserInfo = info parser
  (progDesc "Discard all duplicates that have at least one copy in the specified pathspec's")

parser :: Parser Options
parser = Options
  <$> switch
      ( long "stdin"
     <> help "Read pathspec's from STDIN" )
  <*> many
      ( argument str (metavar "PATHSPEC") )

run :: Options -> IO ()
run opt = do
  pathspecs <- runT $ pathspecSource (optPaths opt) (optStdin opt)
  buckets <- runStoreOp bucketsOp
  runT_ $ source buckets ~> filterBuckets pathspecs ~> autoM putStrLn

type PathSpec = FilePath

filterBuckets :: [PathSpec] -> Process Bucket FilePath
filterBuckets specs = repeatedly $ do
  (Bucket _ pathSet) <- await
  let (matches, others) = Set.partition (matchSpecs specs) $ Set.map unPathKey pathSet
  if (Set.null matches && not (Set.null others))
    then return ()<commit_msg>Remove unused imports in Command.Dupes.Keep<commit_after>{-# LANGUAGE Rank2Types #-}

module Command.Dupes.Keep (
    Options
  , parserInfo
  , run
) where

import Command.ParseUtil
import Dupes
import Store.LevelDB

import Data.List ( isPrefixOf )
import qualified Data.Set as Set
import Data.Machine hiding ( run )
import Options.Applicative

data Options = Options
  { optStdin :: Bool
  , optPaths :: [FilePath]  }

parserInfo :: ParserInfo Options
parserInfo = info parser
  (progDesc "Discard all duplicates that have at least one copy in the specified pathspec's")

parser :: Parser Options
parser = Options
  <$> switch
      ( long "stdin"
     <> help "Read pathspec's from STDIN" )
  <*> many
      ( argument str (metavar "PATHSPEC") )

run :: Options -> IO ()
run opt = do
  pathspecs <- runT $ pathspecSource (optPaths opt) (optStdin opt)
  buckets <- runStoreOp bucketsOp
  runT_ $ source buckets ~> filterBuckets pathspecs ~> autoM putStrLn

type PathSpec = FilePath

filterBuckets :: [PathSpec] -> Process Bucket FilePath
filterBuckets specs = repeatedly $ do
  (Bucket _ pathSet) <- await
  let (matches, others) = Set.partition (matchSpecs specs) $ Set.map unPathKey pathSet
  if (Set.null matches && not (Set.null others))
    then return ()
{'fraction_non_alphanumeric': 0.09735449735449736, 'fraction_numerical': 0.0007054673721340388, 'mean_word_length': 3.9493891797556717, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2455817', 'n_tokens_mistral': 908, 'n_tokens_neox': 870, 'n_words': 293}