<commit_before>from __future__ import absolute_import, division, print_function
from future import standard_library; standard_library.install_aliases()
import logging
_L = logging.getLogger(__name__)

from urllib.parse import urlparse
from os.path import join, basename, dirname, exists, splitext, relpath
from shutil import copy, move, rmtree
from argparse import ArgumentParser
from os import mkdir, rmdir
import tempfile, json, csv

from . import cache, conform, ConformResult

def process(source, destination, extras=dict()):
    ''' Process a single source and destination, return path to JSON state file.
    
        Creates a new directory and files under destination.
    '''
    temp_dir = tempfile.mkdtemp(prefix='process_one-', dir=destination)
    temp_src = join(temp_dir, basename(source))
    copy(source, temp_src)
    
    #
    # Cache source data.
    #
    cache_result = cache(temp_src, temp_dir, extras)
    
    if not cache_result.cache:
        _L.warning('Nothing cached')
        return write_state(source, destination, cache_result, ConformResult.empty())
    
    _L.info('Cached data in {}'.format(cache_result.cache))

    #
    # Conform cached source data.
    #
    conform_result = conform(temp_src, temp_dir, cache_result.todict())
    
    if not conform_result.path:
        _L.warning('Nothing processed')
    else:<commit_msg>Fix bug in state writing if the file failed to download. I introduced this bug in e7fc483ccf9b; this code path is not tested.<commit_after>from __future__ import absolute_import, division, print_function
from future import standard_library; standard_library.install_aliases()
import logging
_L = logging.getLogger(__name__)

from urllib.parse import urlparse
from os.path import join, basename, dirname, exists, splitext, relpath
from shutil import copy, move, rmtree
from argparse import ArgumentParser
from os import mkdir, rmdir
import tempfile, json, csv

from . import cache, conform, ConformResult

def process(source, destination, extras=dict()):
    ''' Process a single source and destination, return path to JSON state file.
    
        Creates a new directory and files under destination.
    '''
    temp_dir = tempfile.mkdtemp(prefix='process_one-', dir=destination)
    temp_src = join(temp_dir, basename(source))
    copy(source, temp_src)
    
    #
    # Cache source data.
    #
    cache_result = cache(temp_src, temp_dir, extras)
    
    if not cache_result.cache:
        _L.warning('Nothing cached')
        return write_state(source, destination, cache_result, ConformResult.empty(), temp_dir)
    
    _L.info('Cached data in {}'.format(cache_result.cache))

    #
    # Conform cached source data.
    #
    conform_result = conform(temp_src, temp_dir, cache_result.todict())
    
    if not conform_result.path:
        _L.warning('Nothing processed')
    else:
{'fraction_non_alphanumeric': 0.08473389355742297, 'fraction_numerical': 0.0017507002801120449, 'mean_word_length': 3.858843537414966, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3914028', 'n_tokens_mistral': 840, 'n_tokens_neox': 833, 'n_words': 260}