<commit_before># -*- coding: utf-8 -*-

import csv
import sys

from sunshine import (
    CONVENTION_HEADERS,
    build_qualification,
    build_address,
    build_rpps,
    clean_text,
    build_eta,
    build_name,
    build_origin
)


def build_programme(row):
    nature = " ".join((row["conv_manifestation_nom"], row["conv_date_debut"],
                       row["conv_date_fin"], row["conv_manifestation_date"]))
    return clean_text(nature)


def process_csv(filepath, output):
    if sys.version_info[0] < 3:
        infile = open(filepath, 'rb')
    else:
        infile = open(filepath, 'r', newline='', encoding='utf8')

    with infile as csvfile:
        reader = csv.DictReader(csvfile, delimiter=';', quotechar='"')

        with open(output, "w") as outputfile:
            outputfile.write(",".join(CONVENTION_HEADERS) + "\n")

            for row in reader:
                cleaned_row = dict((k, clean_text(v)) for k, v in row.items())
                outputfile.write(",".join((build_origin(cleaned_row), cleaned_row["denomination_sociale"],
                                           build_name(cleaned_row),
                                           build_address(cleaned_row), build_qualification(cleaned_row),
                                           build_rpps(cleaned_row), cleaned_row["conv_date_signature"],
                                           cleaned_row["conv_objet"], build_programme(cleaned_row),
                                           build_eta(cleaned_row), cleaned_row["benef_codepostal"])) + "\n")
<commit_msg>Meilleure gestion des dates venant du fichier Etalab<commit_after># -*- coding: utf-8 -*-

import csv
import sys

from sunshine import (
    CONVENTION_HEADERS,
    build_qualification,
    build_address,
    build_rpps,
    clean_text,
    build_eta,
    build_name,
    build_origin
)
from utils import str2date

def build_programme(row):
    nature = " ".join((row["conv_manifestation_nom"], row["conv_date_debut"],
                       row["conv_date_fin"], row["conv_manifestation_date"]))
    return clean_text(nature)


def process_csv(filepath, output):
    if sys.version_info[0] < 3:
        infile = open(filepath, 'rb')
    else:
        infile = open(filepath, 'r', newline='', encoding='utf8')

    with infile as csvfile:
        reader = csv.DictReader(csvfile, delimiter=';', quotechar='"')

        with open(output, "w") as outputfile:
            outputfile.write(",".join(CONVENTION_HEADERS) + "\n")

            for row in reader:
                cleaned_row = dict((k, clean_text(v)) for k, v in row.items())
                outputfile.write(",".join((build_origin(cleaned_row), cleaned_row["denomination_sociale"],
                                           build_name(cleaned_row),
                                           build_address(cleaned_row), build_qualification(cleaned_row),
                                           build_rpps(cleaned_row), str2date(cleaned_row["conv_date_signature"]),
                                           cleaned_row["conv_objet"], build_programme(cleaned_row),
                                           build_eta(cleaned_row), cleaned_row["benef_codepostal"])) + "\n")

{'fraction_non_alphanumeric': 0.11628633948108784, 'fraction_numerical': 0.0031259768677711786, 'mean_word_length': 2.1904287138584246, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3507979', 'n_tokens_mistral': 946, 'n_tokens_neox': 911, 'n_words': 173}