<jupyter_start><jupyter_text>The casefold() method is removes all case distinctions present in a string. It is used for caseless matching, i.e. ignores cases when comparing. For example, German lowercase letter ß is equivalent to ss. However, since ß is already lowercase, lower() method does nothing to it. But, casefold() converts it to ss<jupyter_code>CharsSet = "ascii" # The Character set to be used as the default one when interpreting texts
casefold = lambda u: unicode(icu.UnicodeString(u).foldCase()).encode(CharsSet, "ignore")
def getWordnetPos(_treebank_tag):
    """
    Translate the tree bank PoS tags to the WordNet's
    
    > Parameters:
        _treebank_tag : str     | The tag to be translated
    
    > Returns:
        The relevant WordNet PoS tag
    https://stackoverflow.com/a/15590384/3429115
    """
    if _treebank_tag.startswith('J'):
        return wordnet.ADJ
    elif _treebank_tag.startswith('V'):
        return wordnet.VERB
    elif _treebank_tag.startswith('N'):
        return wordnet.NOUN
    elif _treebank_tag.startswith('R'):
        return wordnet.ADV
    else:
        return '' # CAUTION! It will remove all the words but the four above! implicit stopwords and punctioation removal somehow

def lemmatizeTaggedTerms(_tgdSentsMTX, _isFlattened = True, _oddTokensBehaviour = 3):
    """
    Infer the lemmatized form of tokens with their PoS tags.
    
    > Parameters:
        _tgdSentsMTX : collection of pairs: token - PoS tag | The Token/Tag collection from which we want to find lemmas
        
        _isFlattened : bool                                 | True if the collection is an array rather than a matrix
        
        _oddTokensBehaviour : bool                          | What to do when an odd non-English or non-lingual token is encountered,
                                                            | if 1, such token is not lemmatized and included as-is,
                                                            | if 2, such token is  to be in ASCII form, ignoring non-ascii chars,
                                                            | if 3, such token is included after replacing the odd chars with '?'.
                                                            | 0 (and otherwise), such token is discarded.
                                                            
    > Returns:
        The lemmatized list of tokens according to the selected behaviour.
    
    """
    lemmatizer = WordNetLemmatizer()
    lemmatizedSentsMTX = []
    for pair in _tgdSentsMTX:
        WordNetTag = getWordnetPos(str(pair[1]));
        if (len(WordNetTag) > 0): # Ensure there is a mapping to WordNet categories, ignore punctuations, propositions, determinants, etc.
            # Append the lemmatized token to the sentence list after decoding foreign letters
            # If lemmatizing fails, often due to non-English characters, kepp the word as it is:
            try:
                lemma = lemmatizer.lemmatize(pair[0], WordNetTag)
                lemmatizedSentsMTX.append(lemma)
            except UnicodeDecodeError: # This code won't be entered at all, since the controlle will convert the unicode to the ASCII chars only
                if (_oddTokensBehaviour == 1):
                    lemmatizedSentsMTX.append(pair[0])
                elif (_oddTokensBehaviour == 2):
                    lemmatizedSentsMTX.append(lemmatizer.lemmatize(unicode(pair[0], errors="ignore").encode(CharsSet,"ignore"), WordNetTag))
                elif (_oddTokensBehaviour == 3):
                    lemmatizedSentsMTX.append(lemmatizer.lemmatize(unicode(pair[0], errors="replace").encode(CharsSet,"replace"), WordNetTag))
                        
    return lemmatizedSentsMTX
def tagPoS(_text):
    """
    TagPoS will find the PoS tags for a review, which is of a multiple sentences, preserving the sentences' boundaries.    
    
    > Parameters:
        _text : str                 | the text of the review in str format.

    > Returns:
    the returned object is a mere array of words' tags, but the sentences boundaries would have been 
    taken into consideration nevertheless.
    
        Additionally, _removePunt will be taken into consideration as well.
    
    > Comments:
        As said, it is NOT recommended to remove the punctuation blindly. However, this ability is offered. A better way is to remove 
    the punctuation from the output depending on smart PoS tagging, where the tag is "." === a punctuation.
    
        Also, It is not recommended to remove stop words before this stage, the outputs will contain all part of speeches, and afterwards 
    we can remove everything but adjectives, nouns, verbs, and adverbs, a smarter way to get the essence of a sentence.
    
        The main function here is documented on: http://www.nltk.org/api/nltk.tag.html#nltk.tag.pos_tag_sents
    """
    #if not(isinstance(_text, unicode)):
        #_text = unicode(_text, errors="ignore");        
    listSentences = nltk.sent_tokenize(_text);
    # But even sentences need to be an array of words, so we have to tokenise further, making each sentence array distinguishable by rows
    # Convert the list of sentences to a list of list of words:
    matrixSentences = [];
    for sentence in listSentences:
        if not (isinstance(sentence, str)):
            sentence = sentence.encode(CharsSet,"ignore");            
        sentence = sentence.translate(None, string.punctuation)
        # Append non-empty sentences:
        if (len(sentence) > 0):
            # Back to unicode:
            sentence = unicode(sentence, errors="ignore")
            # bigram[nltk.word_tokenize(sentence)] will place _ with in multiword phrases if present, any. 
            #matrixSentences.append(bigram[nltk.word_tokenize(sentence)])
            matrixSentences.append(nltk.word_tokenize(sentence))
    # Now let's try to PoS on the sentences of the text:
    taggedTokens = nltk.pos_tag_sents(matrixSentences)
    taggedTokens = list(itertools.chain.from_iterable(taggedTokens))
    return taggedTokens
def lemmatizeText(_text, _oddCharsBehaviour = 1):
    """
    The main controller to lemmatize a text and return the list of lemmatized tokens.
    
    This controller takes care of: replacing foreign accents with most likely letters, casefolding, 
    removing newlines and tabs, replacing " with ', removing punctuation after expanding abbreviations 
    for the sake of a better effectiveness, stopping, tokenizing, generating the PoS tags, and 
    generating lemmas of the text depending on PoS tags.
    lemmatizeTaggedTerms
    :parameters:
        _text : unicode             | The text to tokenize and lemmatize, in unicode, which will be converted to str 
                                    | after proper processing
        
        _enAbbrvDict : dictionary   | Holds the english abbreviations' shorthands and expansions, used when 
                                    | removing punctuation
        
        _oddCharsBehaviour : bool   | What to do when an odd non-English or non-lingual token is encountered,
                                    | if 1, such token is not lemmatized and included as-is,
                                    | if 2, such token is  to be in ASCII form, ignoring non-ascii chars,
                                    | if 3, such token is included after replacing the odd chars with '?'.
                                    | 0 (and otherwise), such token is discarded.
    
    :returns:
        A list of lemmatized tokens which belong to the inputted _text and semantic orientation(SO) score
    """    
    # First, strip the unicode of accents, replace with ß with ss
    _text = unidecode.unidecode(_text)
    taggedText = tagPoS(casefold(_text))
    return lemmatizeTaggedTerms(taggedText, _oddTokensBehaviour = _oddCharsBehaviour)
def intensify_polarity(is_customer, row, col, pole):
    if is_customer:
        row[col+'_polarity'] = 'extremely_'+pole
    else:
        row[col+'_polarity'] = pole
    return row
def get_polarity(row, col, is_customer):
    if (row[col+'_textblob_SO'] > 0 and row[col+'_pattern_SO'] > 0) or \
        (row[col+'_textblob_SO'] == 0 and row[col+'_pattern_SO'] > 0) or \
        (row[col+'_textblob_SO'] > 0 and row[col+'_pattern_SO'] == 0):
        row = intensify_polarity(is_customer, row, col, 'positive')
        
    elif (row[col+'_textblob_SO'] < 0 and row[col+'_pattern_SO'] < 0) or \
        (row[col+'_textblob_SO'] == 0 and row[col+'_pattern_SO'] < 0) or \
        (row[col+'_textblob_SO'] < 0 and row[col+'_pattern_SO'] == 0):
        row = intensify_polarity(is_customer, row, col, 'negative')
        
    elif row[col+'_textblob_SO'] == 0 and row[col+'_pattern_SO'] == 0:
        row[col+'_polarity'] = 'neutral'
    else:
        if abs(row[col+'_textblob_SO']) > abs(row[col+'_pattern_SO']):
            if row[col+'_textblob_SO'] > 0:
                row = intensify_polarity(is_customer, row, col, 'positive')
            else:
                row = intensify_polarity(is_customer, row, col, 'negative')
        else:
            if row[col+'_pattern_SO'] > 0:
                row = intensify_polarity(is_customer, row, col, 'positive')
            else:
                row = intensify_polarity(is_customer, row, col, 'negative')
    return row
def apply_lemmatization(row, textual_columns, users_df, intense_bugnotes_list, analyser):
    """
    apply lematizeText Function on every row of dataset.
    
    :parameters:
        row : pandas Dataframe row | row to be processed

    :returns:
        processed row
    """
    try:
        if users_df.loc[row['bug_note_reporter_id'], 'roles'] in ['Customers', 'Customers Suppliers', 'initOS Suppliers', 
                                                                 'Partner']:
            is_customer = True
        else:
            is_customer = False
    except:
        is_customer = False
    for col in textual_columns:
        # Remove hyperlinks
        _text = re.sub(regex_urls, " ", row[col])
        _text = re.sub(regex_emails, " ", _text)
        _text = unicode(_text, "utf-8")
        tb_so = TextBlob(_text)
        row[col+'_textblob_SO'] = tb_so.sentiment.polarity
        #row[col+'_textblob_subjectivity'] = tb_so.sentiment.subjectivity
        pt_so = sentiment(_text)
        row[col+'_pattern_SO'] = pt_so[0]
        #row[col+'_pattern_subjectivity'] = pt_so[1]
        row[col+'_vader_SO'] = analyser.polarity_scores(_text.encode('utf-8'))
        if row['bugnote_id'] in intense_bugnotes_list or is_customer:
            row[col+'_intensify'] = True
            #row[col+'_polarity'] = 'extremely_negative'
        #else:
        #    row = get_polarity(row, col, is_customer)
        row[col] = lemmatizeText(_text)
    return row
def pre_process_data(df, textual_columns, users_df):
    """
        remove the null values and then apply apply_lemmatization.

        :parameters:
            df : pandas Dataframe Object | Dataframe to be processed
    
        :returns:
            Processed Dataframe

    """
    df = df.fillna('')
    df['date_submitted'] = pd.to_datetime(df['date_submitted'], unit='s')
    df['last_updated'] = pd.to_datetime(df['last_updated'], unit='s')
    df['running_duration'] = (df['last_updated'] - df['date_submitted']).astype('timedelta64[D]')
    df.loc[df['running_duration'] > 270, 'pm_ticket'] = True
    df = df.apply(lambda row: apply_lemmatization(row, textual_columns, users_df),axis=1)
    return df
if __name__ == '__main__':
    df = pd.read_csv('../datasets/mantis_bugs_en.csv')
    users_df = pd.read_csv('../datasets/mantis_user_groups.csv', index_col='user_id')
    preprocessed = pre_process_data(df, ['additional_information', 'description', 'summary'], users_df)
    preprocessed.to_csv('../datasets/lexical_semantic_preprocessed_mantis_bugs.csv', encoding='utf-8', index=False)
preprocessed.drop(['additional_information_translation_status',  u'build', u'category', u'category_id',
        u'due_date', u'duplicate_id', u'eta', u'fixed_in_version', u'id_y', u'os',
       u'os_build', u'platform', u'profile_id', u'description_translation_status', u'project_description',
        u'projection', u'sponsorship_total', u'steps_to_reproduce_translation_status',  u'summary_translation_status',
         u'target_version', u'version', u'view_state',  u'sticky', 'bug_text_id', 'name'], axis=1, inplace=True)
df = pd.read_csv('../datasets/lexical_semantic_preprocessed_mantis_bugs_less_columns.csv')
df_relations = pd.read_csv('../datasets/mantis_bugs_relationships.csv') 
df_relations = df_relations[df_relations['source_bug_id'].isin(df['id'])]
bug_counts = df_relations.groupby('source_bug_id')['source_bug_id'].count()
bug_counts = bug_counts.to_frame('freq')
pm_bug_ids = bug_counts[bug_counts['freq']>10].index
df.loc[df['id'].isin(pm_bug_ids), 'pm_ticket'] = True
df.drop(['additional_information_translation_status',  u'build', u'category', u'category_id',
        u'due_date', u'duplicate_id', u'eta', u'fixed_in_version', u'id_y', u'os',
       u'os_build', u'platform', u'profile_id', u'description_translation_status', u'project_description',
        u'projection', u'sponsorship_total', u'steps_to_reproduce_translation_status',  u'summary_translation_status',
         u'target_version', u'version', u'view_state',  u'sticky', 'bug_text_id', 'name'], axis=1, inplace=True)
df.to_csv('../datasets/lexical_semantic_preprocessed_mantis_bugs_less_columns.csv', encoding='utf-8', index=False)
df.loc[df['id'].isin(pm_bug_ids), 'pm_ticket'] = True
df['severity'].value_counts()
len(df[df['severity']>50])
len(df[df['severity']<50])
df[df['severity']>50]['priority'].value_counts()
df[df['severity']<50]['priority'].value_counts()
len(df)
df['severity'].value_counts()
df['priority'].value_counts()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10195540803809798, 'fraction_numerical': 0.005411645861894797, 'mean_word_length': 2.7786259541984735, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 25, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 23, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '77670', 'license': 'MIT', 'n_tokens_mistral': 4251, 'n_tokens_neox': 4039, 'n_words': 1127, 'path': '1) preprocessing/.ipynb_checkpoints/lexical_semantic_preprocessing_bugs-checkpoint.ipynb', 'repo_name': 'initOS/research-criticality-identification', 'size': '13859'}