<commit_before>/*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

use crate::{
    config::{ArtifactPersister, PersistConfig},
    errors::BuildProjectError,
    Artifact, ArtifactContent,
};
use lazy_static::lazy_static;
use log::debug;
use md5::{Digest, Md5};
use regex::Regex;
use std::{fs, path::PathBuf};

lazy_static! {
    static ref RELAY_HASH_REGEX: Regex = Regex::new(r#"@relayHash (\w{32})\n"#).unwrap();
    static ref REQUEST_ID_REGEX: Regex = Regex::new(r#"@relayRequestID (\w+)\n"#).unwrap();
}

pub async fn persist_operations(
    artifacts: &mut [Artifact],
    root_dir: &PathBuf,
    persist_config: &PersistConfig,
    artifact_persister: &Box<dyn ArtifactPersister + Send + Sync>,
) -> Result<(), BuildProjectError> {
    let handles = artifacts.iter_mut().map(|artifact| async move {
        if let ArtifactContent::Operation {
            ref text,
            ref mut id_and_text_hash,
            ..
        } = artifact.content
        {
            let text_hash = md5(text);
            let path = root_dir.join(&artifact.path);
            let persist_id = if let Some(extracted_id) = extract_persist_id(&path, &text_hash) {
                extracted_id
            } else {
                artifact_persister
                    .persist_artifact(text.to_string(), persist_config)
                    .await?
            };
            *id_and_text_hash = Some((persist_id, text_hash));
        }
        Ok(())
    });

    debug!("persisting {} documents", handles.len());
    let results = futures::future::join_all(handles).await;
    debug!("done persisting");
    let errors = results
        .into_iter()
        .filter_map(Result::err)
        .collect::<Vec<_>>();
    if !errors.is_empty() {
        return Err(BuildProjectError::PersistErrors { errors });
    }
    Ok(())
}

fn extract_persist_id(path: &PathBuf, text_hash: &str) -> Option<String> {
    let content = fs::read_to_string(path).ok()?;

    // Looks like a merge conflict, let's not trust this file.
    if content.contains("<<<<") || content.contains(">>>>") {
        return None;
    }

    // If the existing hash doesn't match the hash of the new query test, abort.
    if let Some(existing_hash) = extract_relay_hash(&content) {<commit_msg>only count operations we couldn't recover from disk<commit_after>/*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

use crate::{
    config::{ArtifactPersister, PersistConfig},
    errors::BuildProjectError,
    Artifact, ArtifactContent,
};
use lazy_static::lazy_static;
use log::debug;
use md5::{Digest, Md5};
use regex::Regex;
use std::{fs, path::PathBuf};

lazy_static! {
    static ref RELAY_HASH_REGEX: Regex = Regex::new(r#"@relayHash (\w{32})\n"#).unwrap();
    static ref REQUEST_ID_REGEX: Regex = Regex::new(r#"@relayRequestID (\w+)\n"#).unwrap();
}

pub async fn persist_operations(
    artifacts: &mut [Artifact],
    root_dir: &PathBuf,
    persist_config: &PersistConfig,
    artifact_persister: &Box<dyn ArtifactPersister + Send + Sync>,
) -> Result<(), BuildProjectError> {
    let handles = artifacts
        .iter_mut()
        .flat_map(|artifact| {
            if let ArtifactContent::Operation {
                ref text,
                ref mut id_and_text_hash,
                ..
            } = artifact.content
            {
                let text_hash = md5(text);
                let artifact_path = root_dir.join(&artifact.path);
                if let Some(id) = extract_persist_id(&artifact_path, &text_hash) {
                    *id_and_text_hash = Some((id, text_hash));
                    None
                } else {
                    let text = text.clone();
                    Some(async move {
                        artifact_persister
                            .persist_artifact(text, persist_config)
                            .await
                            .map(|id| {
                                *id_and_text_hash = Some((id, text_hash));
                            })
                    })
                }
            } else {
                None
            }
        })
        .collect::<Vec<_>>();
    debug!("persisting {} documents", handles.len());
    let results = futures::future::join_all(handles).await;
    debug!("done persisting");
    let errors = results
        .into_iter()
        .filter_map(Result::err)
        .collect::<Vec<_>>();
    if !errors.is_empty() {
        return Err(BuildProjectError::PersistErrors { errors });
    }
    Ok(())
}

fn extract_persist_id(path: &PathBuf, text_hash: &str) -> Option<String> {
    let content = fs::read_to_string(path).ok()?;

    // Looks like a merge conflict, let's not trust this file.
    if content.contains("<<<<") || content.contains(">>>>") {
        return None;
    }

    // If the existing hash doesn't match the hash of the new query test, abort.
    if let Some(existing_hash) = extract_relay_hash(&content) {
{'fraction_non_alphanumeric': 0.12986512524084778, 'fraction_numerical': 0.0019267822736030828, 'mean_word_length': 2.295873015873016, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2770605', 'n_tokens_mistral': 1584, 'n_tokens_neox': 1547, 'n_words': 374}