<commit_before>#!/usr/bin/python

import urllib
import json
import sys
import csv
import re


# Convert from CPC IPC ref format A63H17/273
# to IPC symbol format A63H0017273000 (zero padded, 4 digit for main
# group, 6 digits for group)
def cpcRefToIpc(s):
    pos_subcode = s.find('/')
    if pos_subcode > 0:
        group = '0000' + s[4:pos_subcode]
        group = group[-4:]
        subgroup = s[pos_subcode + 1:] + '000000'
        subgroup = subgroup[0:6]
        return s[0:4] + group + subgroup
    elif len(s) > 4:
        group = '0000' + s[4:]
        group = group[-4:]
        return s[0:4] + group + '000000'
    else:
        return s

# Get IPC context and, if exists, return IPC id


def CheckIPCContext(s):
    code = cpcRefToIpc(s.replace(" ", ""))
    serviceurl_ipc = 'http://localhost:8080/pat-clas-service/rest/v1.0/IPC/ancestorsAndSelf?symbol='

    url_ipc = serviceurl_ipc + code
    # we check IPC
    uh = urllib.urlopen(url_ipc)
    data = uh.read()
    js = json.loads(data.decode("ascii", "ignore"))
    position = {}
    ipc_desc = {}
    ipc_hrchy = {}
    ipc = {}
    retval = {}
    full_descr = ' '
    ipc_desc['ipc_code'] = ipc['ipc_code'] = ipc_hrchy['ipc_code'] = s
    position['ipc_version'] = ipc_desc['ipc_version'] = ipc[
        'ipc_version'] = ipc_hrchy['ipc_version'] = '2016.01'

    if js:
        # getting the level of the code
        ipc_desc['level'] = -1
        for elt in js:
            if elt['symbol'] == code:
                ipc_desc['level'] = elt['level']
                ipc['description'] = elt['textBody'] + '.'

        for elt in js:
            if elt['level'] <= ipc_desc['level']:
                if elt['level'] in (0, 1, 2):
                    if elt['level'] == 0:
                        ipc_hrchy['parent'] = elt['symbol']
                        position['section'] = cleanTitles(elt['textBody'][10:])
                    if elt['level'] == 1:
                        position['class'] = cleanTitles(elt['textBody'])
                    if elt['level'] == 2:
                        position['ipc_position'] = ipc_desc[
                            'ipc_position'] = elt['symbol']
                        position['subclass'] = cleanTitles(elt['textBody'])
                        position['full_subclass'] = re.sub(
                            '\n', ' ', elt['textBody'])
                else:

                    if elt['textBody'][0].isupper():
                        full_descr += '. ' + elt['textBody']
                    else:
                        full_descr += ' ' + elt['textBody']
                    if elt['level'] == ipc_desc['level'] - 1:
                        ipc_hrchy['ancestor'] = elt['symbol']

    if full_descr != ' ':
        full_descr = re.sub('\n', ' ', full_descr)
        ipc_desc['ipc_desc'] = full_descr[3:] + '.'

    retval[0] = position
    retval[1] = ipc_desc
    retval[2] = ipc
    retval[3] = ipc_hrchy
    return retval


def dicToCsv(output_file, cvs_columns, dict_data):
    try:
        with open(ofname, 'w') as csvfile:
            writer = csv.DictWriter(csvfile, fieldnames=csv_columns,
                                    delimiter='\t', quotechar='"', quoting=csv.QUOTE_MINIMAL)
            writer.writeheader()
            for data in dict_data:
                writer.writerow(data)
    except IOError as (errno, strerror):
        print("I/O error({0}): {1}".format(errno, strerror))


def cleanTitles(s):
    r = re.sub('\n', ' ', s)
    r = re.sub('[^A-Z ]', '', r)
    r = r.title()
    return r

ifname = '03_01_abstract_from_ipc_input.csv'
dict_data = {'pos': [], 'desc': [], 'hrchy': [], 'ipc': []}
<commit_msg>Clean text separating ideas and deleting unnecessary punctuation marks<commit_after>#!/usr/bin/python

import urllib
import json
import sys
import csv
import re


# Convert from CPC IPC ref format A63H17/273
# to IPC symbol format A63H0017273000 (zero padded, 4 digit for main
# group, 6 digits for group)
def cpcRefToIpc(s):
    pos_subcode = s.find('/')
    if pos_subcode > 0:
        group = '0000' + s[4:pos_subcode]
        group = group[-4:]
        subgroup = s[pos_subcode + 1:] + '000000'
        subgroup = subgroup[0:6]
        return s[0:4] + group + subgroup
    elif len(s) > 4:
        group = '0000' + s[4:]
        group = group[-4:]
        return s[0:4] + group + '000000'
    else:
        return s

# Get IPC context and, if exists, return IPC id


def CheckIPCContext(s):
    code = cpcRefToIpc(s.replace(" ", ""))
    serviceurl_ipc = 'http://localhost:8080/pat-clas-service/rest/v1.0/IPC/ancestorsAndSelf?symbol='

    url_ipc = serviceurl_ipc + code
    # we check IPC
    uh = urllib.urlopen(url_ipc)
    data = uh.read()
    js = json.loads(data.decode("ascii", "ignore"))
    position = {}
    ipc_desc = {}
    ipc_hrchy = {}
    ipc = {}
    retval = {}
    full_descr = ' '
    ipc_desc['ipc_code'] = ipc['ipc_code'] = ipc_hrchy['ipc_code'] = s
    position['ipc_version'] = ipc_desc['ipc_version'] = ipc[
        'ipc_version'] = ipc_hrchy['ipc_version'] = '2016.01'

    if js:
        # getting the level of the code
        ipc_desc['level'] = -1
        for elt in js:
            if elt['symbol'] == code:
                ipc_desc['level'] = elt['level']
                ipc['description'] = elt['textBody'] + '.'

        for elt in js:
            if elt['level'] <= ipc_desc['level']:
                if elt['level'] in (0, 1, 2):
                    if elt['level'] == 0:
                        ipc_hrchy['parent'] = elt['symbol']
                        position['section'] = cleanTitles(elt['textBody'][10:])
                    if elt['level'] == 1:
                        position['class'] = cleanTitles(elt['textBody'])
                    if elt['level'] == 2:
                        position['ipc_position'] = ipc_desc[
                            'ipc_position'] = elt['symbol']
                        position['subclass'] = cleanTitles(elt['textBody'])
                        position['full_subclass'] = re.sub(
                            '\n', ' ', elt['textBody'])
                else:
                    full_descr += ' ' + elt['textBody']
                    if elt['level'] == ipc_desc['level'] - 1:
                        ipc_hrchy['ancestor'] = elt['symbol']

    if full_descr != ' ':
        full_descr = re.sub('\n', ' ', full_descr)
        full_descr = addDot(full_descr)
        ipc_desc['ipc_desc'] = full_descr[4:] + '.'
        ipc_desc['ipc_desc'] = cleanText(ipc_desc['ipc_desc'])
    retval[0] = position
    retval[1] = ipc_desc
    retval[2] = ipc
    retval[3] = ipc_hrchy
    return retval


def dicToCsv(output_file, cvs_columns, dict_data):
    try:
        with open(ofname, 'w') as csvfile:
            writer = csv.DictWriter(csvfile, fieldnames=csv_columns,
                                    delimiter='\t', quotechar='"', quoting=csv.QUOTE_MINIMAL)
            writer.writeheader()
            for data in dict_data:
                writer.writerow(data)
    except IOError as (errno, strerror):
        print("I/O error({0}): {1}".format(errno, strerror))


def cleanTitles(s):
    r = re.sub('\n', ' ', s)
    r = re.sub('[^A-Z ]', '', r)
    r = r.title()
    return r


def cleanText(s):    
    cleaned = s.replace(", ,", "")
    cleaned = cleaned.replace(",,", ",")
    cleaned = cleaned.replace(" ,", ",")
    cleaned = cleaned.replace(". .", "")
    cleaned = cleaned.replace("..", ".")
    cleaned = cleaned.replace(" . ", ". ")
    cleaned = re.sub("\s\s+", " ", cleaned)
    cleaned = cleaned.replace(",.", ".")
    cleaned = cleaned.replace(".,", ".")
    cleaned = cleaned.replace(" .", ".")
    cleaned = cleaned.replace(" ,", ",")
    return cleaned


def addDot(s):
    result = re.sub(ur"(?!^)(?=\s[A-Z])", ". ", s)
    return result

ifname = '03_01_abstract_from_ipc_input.csv'
dict_data = {'pos': [], 'desc': [], 'hrchy': [], 'ipc': []}

{'fraction_non_alphanumeric': 0.1616698292220114, 'fraction_numerical': 0.022137887413029727, 'mean_word_length': 1.9856495468277946, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2493228', 'n_tokens_mistral': 2850, 'n_tokens_neox': 2661, 'n_words': 551}