<commit_before>{-# LANGUAGE OverloadedStrings #-}

module Main where

import Network.HTTP.Conduit
import Network.HTTP.Types.Status
import Control.Monad.Error
import Control.Monad.Trans.Resource
import System.FilePath.Posix
import Data.List
import System.Directory
import Control.Exception

import Api
import Root
import qualified Account as A
import qualified Document as D
import qualified File as F
import qualified Config as C
import Oauth
import Http (Session, AccessToken)


guiSync :: Int -> IO (SyncResult ())
guiSync runNumber = runErrorT $ do
    userHome <- liftIO getHomeDirectory
    let config = C.defaultConfig userHome
    liftIO $ F.createSyncDir $ C.syncDir config
    hasLocalChanges <- liftIO $ localChange config
    let fullSync = runNumber `mod` 6 == 0
    if hasLocalChanges || fullSync then
        do
            at <- liftIO loadAccessToken
            token <- case at of
                    Nothing -> throwError NotAuthenticated
                    Just t -> return t
            gsync config token
    else
        return ()


gsync :: C.Config -> AccessToken -> ErrorT SyncError IO ()
gsync config token = do
    res <- liftIO $ try (sync config (Right token))
    case res of
        Left (StatusCodeException (Status 403 _) hdrs _) -> do
            liftIO $ putStrLn $ "403 status" ++ (show hdrs)
            newToken <- refreshAccessToken token --TODO: exceptions?
            liftIO $ storeAccessToken newToken
            gsync config newToken  --TODO: retry count??
        Left (StatusCodeException (Status 401 _) hdrs _) -> do
            liftIO $ putStrLn $ "401 status " ++ (show hdrs)
            throwError NotAuthenticated
        Left e -> throwError $ HttpFailed e
        Right _ -> return ()

localChange :: C.Config -> IO Bool
localChange config = do
    let syncDir = C.syncDir config
    let syncFile = F.syncFile syncDir
    files <- F.existingFiles syncDir
    lastState <- F.readSyncFile syncFile
    return $ not $ lastState == files

sync :: C.Config -> Session -> IO ()
sync config session = runResourceT $ do
    manager <- liftIO $ newManager conduitManagerSettings
    (root, account) <- getAccount manager session
    archiveLink <- liftIO $ linkOrException "document_archive" $ A.link account
    documents <- getDocs manager session archiveLink
    let syncDir = C.syncDir config
    let syncFile = F.syncFile syncDir
    files <- liftIO $ F.existingFiles syncDir
    lastState <- liftIO $ F.readSyncFile syncFile
    let (docsToDownload, newFiles, deletedFiles) = F.syncDiff lastState files documents
    liftIO $ void $ mapM debugLog [
                            "download: [" ++ intercalate ", " (map D.filename docsToDownload) ++ "]",
                            "upload: [" ++ intercalate ", " newFiles ++ "]",
                            "deleted: [" ++ intercalate ", " deletedFiles ++ "]" ]
    downloadAll session manager syncDir docsToDownload
    uploadLink <- liftIO $ linkOrException "upload_document" $ A.link account
    uploadAll session manager uploadLink (csrfToken root) (map (combine syncDir) newFiles)
    liftIO $ F.deleteAll syncDir deletedFiles
    newState <- liftIO $ F.existingFiles syncDir
    liftIO $ F.writeSyncFile syncFile newState
    liftIO $ debugLog "Finished"

debugLog :: String -> IO ()
debugLog = putStrLn<commit_msg>Debug logging<commit_after>{-# LANGUAGE OverloadedStrings #-}

module Main where

import Network.HTTP.Conduit
import Network.HTTP.Types.Status
import Control.Monad.Error
import Control.Monad.Trans.Resource
import System.FilePath.Posix
import Data.List
import System.Directory
import Control.Exception

import Api
import Root
import qualified Account as A
import qualified Document as D
import qualified File as F
import qualified Config as C
import Oauth
import Http (Session, AccessToken)


guiSync :: Int -> IO (SyncResult ())
guiSync runNumber = runErrorT $ do
    userHome <- liftIO getHomeDirectory
    let config = C.defaultConfig userHome
    liftIO $ F.createSyncDir $ C.syncDir config
    hasLocalChanges <- liftIO $ localChange config
    let fullSync = runNumber `mod` 6 == 0
    if hasLocalChanges || fullSync then
        do
            at <- liftIO loadAccessToken
            token <- case at of
                    Nothing -> throwError NotAuthenticated
                    Just t -> return t
            gsync config token
    else
        return ()


gsync :: C.Config -> AccessToken -> ErrorT SyncError IO ()
gsync config token = do
    res <- liftIO $ try (sync config (Right token))
    case res of
        Left (StatusCodeException (Status 403 _) hdrs _) -> do
            liftIO $ debugLog $ "403 status" ++ (show hdrs)
            newToken <- refreshAccessToken token --TODO: exceptions?
            liftIO $ storeAccessToken newToken
            gsync config newToken  --TODO: retry count??
        Left (StatusCodeException (Status 401 _) hdrs _) -> do
            liftIO $ debugLog $ "401 status " ++ (show hdrs)
            throwError NotAuthenticated
        Left e -> throwError $ HttpFailed e
        Right _ -> return ()

localChange :: C.Config -> IO Bool
localChange config = do
    let syncDir = C.syncDir config
    let syncFile = F.syncFile syncDir
    files <- F.existingFiles syncDir
    lastState <- F.readSyncFile syncFile
    return $ not $ lastState == files

sync :: C.Config -> Session -> IO ()
sync config session = runResourceT $ do
    manager <- liftIO $ newManager conduitManagerSettings
    (root, account) <- getAccount manager session
    archiveLink <- liftIO $ linkOrException "document_archive" $ A.link account
    documents <- getDocs manager session archiveLink
    let syncDir = C.syncDir config
    let syncFile = F.syncFile syncDir
    files <- liftIO $ F.existingFiles syncDir
    lastState <- liftIO $ F.readSyncFile syncFile
    let (docsToDownload, newFiles, deletedFiles) = F.syncDiff lastState files documents
    liftIO $ void $ mapM debugLog [
                            "download: [" ++ intercalate ", " (map D.filename docsToDownload) ++ "]",
                            "upload: [" ++ intercalate ", " newFiles ++ "]",
                            "deleted: [" ++ intercalate ", " deletedFiles ++ "]" ]
    downloadAll session manager syncDir docsToDownload
    uploadLink <- liftIO $ linkOrException "upload_document" $ A.link account
    uploadAll session manager uploadLink (csrfToken root) (map (combine syncDir) newFiles)
    liftIO $ F.deleteAll syncDir deletedFiles
    newState <- liftIO $ F.existingFiles syncDir
    liftIO $ F.writeSyncFile syncFile newState
    liftIO $ debugLog "Finished"

debugLog :: String -> IO ()
debugLog = putStrLn
--debugLog _ = return ()
{'fraction_non_alphanumeric': 0.08379553290361265, 'fraction_numerical': 0.004197271773347324, 'mean_word_length': 2.680088251516823, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5027639', 'n_tokens_mistral': 1858, 'n_tokens_neox': 1791, 'n_words': 614}