<commit_before>#!/usr/bin/env python3
# -*-coding:UTF-8 -*

import configparser
import os
import sys
import gzip
import io
import redis
import base64
import datetime
import time

from sflock.main import unpack
import sflock

from Helper import Process
from pubsublogger import publisher

def create_paste(uuid, paste_content, ltags, ltagsgalaxies, name):

    now = datetime.datetime.now()
    save_path = 'submitted/' + now.strftime("%Y") + '/' + now.strftime("%m") + '/' + now.strftime("%d") + '/' + name + '.gz'

    full_path = filename = os.path.join(os.environ['AIL_HOME'],
                            p.config.get("Directories", "pastes"), save_path)

    if os.path.isfile(full_path):
        addError(uuid, 'File: ' + save_path + ' already exist in submitted pastes')
        return 1

    try:
        gzipencoded = gzip.compress(paste_content)
        gzip64encoded = base64.standard_b64encode(gzipencoded).decode()
    except:
        abord_file_submission(uuid, "file error")
        continue

    # send paste to Global module
    relay_message = "{0} {1}".format(save_path, gzip64encoded)
    p.populate_set_out(relay_message, 'Mixer')

    # add tags
    add_tags(ltags, ltagsgalaxies, full_path)

    r_serv_log_submit.incr(uuid + ':nb_end')
    r_serv_log_submit.incr(uuid + ':nb_sucess')

    if r_serv_log_submit.get(uuid + ':nb_end') == r_serv_log_submit.get(uuid + ':nb_total'):
        r_serv_log_submit.set(uuid + ':end', 1)<commit_msg>fix submit paste<commit_after>#!/usr/bin/env python3
# -*-coding:UTF-8 -*

import configparser
import os
import sys
import gzip
import io
import redis
import base64
import datetime
import time

from sflock.main import unpack
import sflock

from Helper import Process
from pubsublogger import publisher

def create_paste(uuid, paste_content, ltags, ltagsgalaxies, name):

    now = datetime.datetime.now()
    save_path = 'submitted/' + now.strftime("%Y") + '/' + now.strftime("%m") + '/' + now.strftime("%d") + '/' + name + '.gz'

    full_path = filename = os.path.join(os.environ['AIL_HOME'],
                            p.config.get("Directories", "pastes"), save_path)

    if os.path.isfile(full_path):
        addError(uuid, 'File: ' + save_path + ' already exist in submitted pastes')
        return 1

    try:
        gzipencoded = gzip.compress(paste_content)
        gzip64encoded = base64.standard_b64encode(gzipencoded).decode()
    except:
        abord_file_submission(uuid, "file error")
        return 1

    # send paste to Global module
    relay_message = "{0} {1}".format(save_path, gzip64encoded)
    p.populate_set_out(relay_message, 'Mixer')

    # add tags
    add_tags(ltags, ltagsgalaxies, full_path)

    r_serv_log_submit.incr(uuid + ':nb_end')
    r_serv_log_submit.incr(uuid + ':nb_sucess')

    if r_serv_log_submit.get(uuid + ':nb_end') == r_serv_log_submit.get(uuid + ':nb_total'):
        r_serv_log_submit.set(uuid + ':end', 1)
{'fraction_non_alphanumeric': 0.12863496407800204, 'fraction_numerical': 0.01128977078344167, 'mean_word_length': 3.4846625766871164, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6468286', 'n_tokens_mistral': 1049, 'n_tokens_neox': 996, 'n_words': 214}