<commit_before># This file will be used to process raw and preprocessed-JSON documents from the scrapers
import os
import json
import datetime
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import logging
sys.path.insert(1, os.path.join(
    os.path.dirname(os.path.abspath(__file__)),
    os.pardir,
))
from api.collision_detection import collision_detector

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)

BASE_DIR = os.path.abspath(os.path.join(
    os.path.dirname(os.path.abspath(__file__)),
    os.pardir,
))


def process_raw(doc, source, doc_id, filetype, consumer_version):
    """
        Takes a document (in the form of text, and saves it to disk
        with the specified name and the designated filetype in the
        specified source directory. It also saves a manifest file
        in the directory, containing the git hash for the version
        of the consumer it was produced with.
    """
    timestamp = datetime.datetime.now()
    directory = '/archive/' + str(source).replace('/', '') + '/' + str(doc_id).replace('/', '') + '/' + str(timestamp) + '/'
    filepath = BASE_DIR + directory + "raw" + '.' + str(filetype)

    dir_path = BASE_DIR
    for dir in directory.split("/"):
        dir_path += dir + "/"
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)

    with open(filepath, 'w') as f:
        try:
            f.write(str(doc))
        except IOError as e:
            logger.log(e)
            return None
    with open(dir_path + 'manifest.json', 'w') as f:
        info = {}
        info['consumer_version'] = consumer_version.strip()
        info['source'] = source.strip()
        info['timestamp'] = str(timestamp)
        f.write(json.dumps(info))

    return timestamp


def process(doc, timestamp):
    """
        Takes a JSON document and extracts the information necessary
        to make an OSF project, then creates that OSF project through
        the API (does not exist yet)
        Format specification:
        {
            'title': {PROJECT_TITLE},
            'contributors: [{PROJECT_CONTRIBUTORS}],
            'properties': {
                {VALID_NODE_PROPERTY}: {NODE_PROPERTY_VALUE},
            },
            'meta': {META_DATA FOR PROJECT | EMPTY},
            'id': {DOI OR UNIQUE ID OF PROJECT},
            'source': {SOURCE OF SCRAPE}
        }
    """
    if None in [doc.get('title'), doc.get('contributors'), doc.get('id'), doc.get('source')]:
        logger.error(": Document sent for processing did not match schema")
        return None

    # Collision detection
    conflict = collision_detector.detect(doc)
    if conflict:
        logger.warn("Document with id {0} and timestamp {1} from source {2} already found in database".format(doc['id'], timestamp, doc['source']))

    directory = '/archive/' + doc['source'].replace('/', '') + '/' + str(doc['id']).replace('/', '') + '/' + str(timestamp) + '/'
    filepath = BASE_DIR + directory + "parsed.json"

    dir_path = BASE_DIR
    for dir in directory.split("/"):
        dir_path += dir + "/"
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)

    with open(filepath, 'w') as f:<commit_msg>Update documentation, and timestamp is now indexed in elasticsearch<commit_after># This file will be used to process raw and preprocessed-JSON documents from the scrapers
import os
import json
import datetime
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import logging
sys.path.insert(1, os.path.join(
    os.path.dirname(os.path.abspath(__file__)),
    os.pardir,
))
from api.collision_detection import collision_detector

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)

BASE_DIR = os.path.abspath(os.path.join(
    os.path.dirname(os.path.abspath(__file__)),
    os.pardir,
))


def process_raw(doc, source, doc_id, filetype, consumer_version):
    """
        Takes a text document and saves it to disk with the specified name and the designated filetype
        in the specified source directory. It also saves a manifest file in the directory, containing
        the git hash for the version of the consumer it was produced with.
    """
    timestamp = datetime.datetime.now()
    directory = '/archive/' + str(source).replace('/', '') + '/' + str(doc_id).replace('/', '') + '/' + str(timestamp) + '/'
    filepath = BASE_DIR + directory + "raw" + '.' + str(filetype)

    dir_path = BASE_DIR
    for dir in directory.split("/"):
        dir_path += dir + "/"
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)

    with open(filepath, 'w') as f:
        try:
            f.write(str(doc))
        except IOError as e:
            logger.log(e)
            return None
    with open(dir_path + 'manifest.json', 'w') as f:
        info = {}
        info['consumer_version'] = consumer_version.strip()
        info['source'] = source.strip()
        info['timestamp'] = str(timestamp)
        f.write(json.dumps(info))

    return timestamp


def process(doc, timestamp):
    """
        Takes a JSON document and extracts the information necessary
        to make an OSF project, then creates that OSF project through
        the API (does not exist yet)
        Format specification:
        {
            'title': {PROJECT_TITLE},
            'contributors: [{PROJECT_CONTRIBUTORS}],
            'properties': {
                {VALID_NODE_PROPERTY}: {NODE_PROPERTY_VALUE},
            },
            'meta': {META_DATA FOR PROJECT | EMPTY},
            'id': {DOI OR UNIQUE ID OF PROJECT},
            'source': {SOURCE OF SCRAPE}
        }
    """
    if None in [doc.get('title'), doc.get('contributors'), doc.get('id'), doc.get('source')]:
        logger.error(": Document sent for processing did not match schema")
        return None

    # Collision detection
    conflict = collision_detector.detect(doc)
    if conflict:
        logger.warn("Document with id {0} and timestamp {1} from source {2} already found in database".format(doc['id'], timestamp, doc['source']))

    directory = '/archive/' + doc['source'].replace('/', '') + '/' + str(doc['id']).replace('/', '') + '/' + str(timestamp) + '/'
    filepath = BASE_DIR + directory + "parsed.json"

    dir_path = BASE_DIR
    for dir in directory.split("/"):
        dir_path += dir + "/"
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)

    doc['timestamp'] = str(timestamp)

    with open(filepath, 'w') as f:
{'fraction_non_alphanumeric': 0.13255240443896424, 'fraction_numerical': 0.0015413070283600493, 'mean_word_length': 2.684838160136286, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1260654', 'n_tokens_mistral': 1946, 'n_tokens_neox': 1911, 'n_words': 547}