<commit_before>import nltk
import networkx as nx

from nltk.corpus import stopwords
from sentenceSelection import tokeniseSentences

STOP = stopwords.words('english')


def tokeniseWords(rawText):
    # Use sentence tokensiser to remove code 'sentences'
    sentList = tokeniseSentences(rawText)
    text = ""
    for sentence in sentList:
        text += sentence + " "

    # Split text into words
    return  nltk.word_tokenize(text)

def removeStop(wordList, stopList):
    goodWords = []
    for word in wordList:
        if word not in stopList:
            goodWords.append(word)
    return goodWords

def filterTags(wordList):
    filteredWords = []
    tagWords = nltk.pos_tag(wordList)
    for word, tag in tagWords:
        if tag =="NN" or tag =="JJ":
            filteredWords.append(word)

    return filteredWords

def extract(wordList, num=4, threshold=5):
    # Construct the graph
    g = nx.DiGraph()
    for w in wordList:
        g.add_node(w)

    for i, w in enumerate(wordList):
        for j in xrange(1, threshold+1):
            try:
                g.add_edge(w, wordList[i+j])
            except IndexError:
                pass

    # Compute values
    pairs = nx.pagerank(g).items()
    sortedPairs = sorted(pairs, key=lambda x: x[1], reverse=True)
    goodKeywords = []
    for keyword, _ in sortedPairs[:num]:
        goodKeywords.append(keyword)

    return goodKeywords

def extractKeywords(rawText, num=4, stopList=STOP, threshold=5):
    wordList = tokeniseWords(rawText)
    wordList = removeStop(wordList, stopList)
    wordList = filterTags(wordList)
    return extract(wordList, threshold, num)<commit_msg>Fixed ordering on function<commit_after>import nltk
import networkx as nx

from nltk.corpus import stopwords
from sentenceSelection import tokeniseSentences

STOP = stopwords.words('english')


def tokeniseWords(rawText):
    # Use sentence tokensiser to remove code 'sentences'
    sentList = tokeniseSentences(rawText)
    text = ""
    for sentence in sentList:
        text += sentence + " "

    # Split text into words
    return  nltk.word_tokenize(text)

def removeStop(wordList, stopList):
    goodWords = []
    for word in wordList:
        if word not in stopList:
            goodWords.append(word)
    return goodWords

def filterTags(wordList):
    filteredWords = []
    tagWords = nltk.pos_tag(wordList)
    for word, tag in tagWords:
        if tag =="NN" or tag =="JJ":
            filteredWords.append(word)

    return filteredWords

def extract(wordList, num=4, threshold=5):
    # Construct the graph
    g = nx.DiGraph()
    for w in wordList:
        g.add_node(w)

    for i, w in enumerate(wordList):
        for j in xrange(1, threshold+1):
            try:
                g.add_edge(w, wordList[i+j])
            except IndexError:
                pass

    # Compute values
    pairs = nx.pagerank(g).items()
    sortedPairs = sorted(pairs, key=lambda x: x[1], reverse=True)
    goodKeywords = []
    for keyword, _ in sortedPairs[:num]:
        goodKeywords.append(keyword)

    return goodKeywords

def extractKeywords(rawText, num=4, stopList=STOP, threshold=5):
    wordList = tokeniseWords(rawText)
    wordList = removeStop(wordList, stopList)
    wordList = filterTags(wordList)
    return extract(wordList, num, threshold)
{'fraction_non_alphanumeric': 0.09307948020550015, 'fraction_numerical': 0.0042308854638863705, 'mean_word_length': 2.866822429906542, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '343764', 'n_tokens_mistral': 1064, 'n_tokens_neox': 1022, 'n_words': 280}