<jupyter_start><jupyter_text>Implementação  Dados dos datasetsDataset | Query | Dataset | Rel | % Rel--- | --- | --- | --- | ---Hall | Defect prediction | 8911 | 104 | 1.17Radjenovic | Defect prediction metrics | 6000 | 48 | 0.80Kitchenham | Literature review | 1704 | 45 | 2.64Wahono | Defect prediction | 7002 | 62 | 0.88  Bibliotecas<jupyter_code>import os
import sys
#import datetime
from collections import OrderedDict
#from random import seed
#from random import randint
import nltk
import pandas as pd
import numpy as np
from nltk.tokenize import RegexpTokenizer
from nltk.stem.porter import PorterStemmer
from nltk.corpus import stopwords
import gensim
from gensim.corpora import Dictionary
from gensim.models.wrappers import LdaMallet
from gensim.models.wrappers.ldamallet import malletmodel2ldamodel
from gensim.models.coherencemodel import CoherenceModel
from gensim.models.phrases import Phrases, Phraser
#from google.colab import drive

print("Python version")
print (sys.version)
print("Gensim version")
print (gensim.__version__)
print("Pandas version")
print (pd.__version__)

nltk.download('stopwords')<jupyter_output>Python version
3.6.10 |Anaconda, Inc.| (default, Mar 25 2020, 23:51:54) 
[GCC 7.3.0]
Gensim version
3.8.0
Pandas version
1.0.3<jupyter_text>Números randômicos<jupyter_code># seed(datetime.datetime.now().second)

random_numbers = [534, 424, 826, 310, 983]

# for _ in range(5):
# 	value = randint(0, 1000)
# 	random_numbers.append(value)
 
print(random_numbers)<jupyter_output>[534, 424, 826, 310, 983]<jupyter_text>Tratamento do dataset<jupyter_code>def get_corpus_for_docs(docs):
    """
    Faz o tratamento da base e retorna o corpus, dicionário e query tratada
    """
    # Download and import stopwords library
    stopwds = stopwords.words('english')

    # Split the documents into tokens.
    tokenizer = RegexpTokenizer(r'\w+')
    for idx in range(len(docs)):
        docs[idx] = str(docs[idx]).lower()  # Convert to lowercase.
        docs[idx] = tokenizer.tokenize(docs[idx])  # Split into words.

#     # Remove numbers, but not words that contain numbers.
#     docs = [[token for token in doc if not token.isnumeric()] for doc in docs]

#     # Remove words that are only one character.
#     docs = [[token for token in doc if len(token) > 1] for doc in docs]        
        
    # Remove stopwords.
    docs = [[token for token in doc if token not in stopwds] for doc in docs]

    # Porter stemmer.
    stemmer = PorterStemmer()
    docs = [[stemmer.stem(token) for token in doc] for doc in docs]
    
    # Compute bigrams.
    phrases = Phrases(docs, min_count=1, threshold=2)
    bigram = Phraser(phrases)
    for idx in range(len(docs)):
        docs[idx] = bigram[docs[idx]]
        
    # text processing is done
    query = docs[-1]
    docs = docs[:-1]
    
    # Remove rare and common tokens.
    # Create a dictionary representation of the documents.
    dictionary = Dictionary(docs)

    # Bag-of-words representation of the documents.
    corpus = [dictionary.doc2bow(doc) for doc in docs]

    return corpus, dictionary, docs, query<jupyter_output><empty_output><jupyter_text>Executa LDA e retorna TOP documentos<jupyter_code>def get_top_documents(df, corpus, dictionary, query, num_topics, random_state, N=50):
    """
    Retorna TOP 10 documentos da query.
    
    Recebe a base original, corpus, dicionário, query de busca, número de
    tópicos e a semente do random.
    """
    # Make a index to word dictionary.
    temp = dictionary[0]  # This is only to "load" the dictionary.
    id2word = dictionary.id2token

    # Train LDA model.
    path_to_mallet_binary = os.path.join(os.getcwd(), "mallet-2.0.8", "bin", "mallet")
    mallet_model = LdaMallet(
        path_to_mallet_binary,
        corpus=corpus,
        id2word=id2word,
        num_topics=num_topics,
        random_seed=random_state
    )
    model = malletmodel2ldamodel(mallet_model)

    # top_topics = model.top_topics(corpus, topn=10)

    # Topic coherence
    texts = [[dictionary[word_id] for word_id, freq in doc] for doc in corpus]
    cm = CoherenceModel(model=model, corpus=corpus, texts=texts, dictionary=dictionary, coherence='c_v')
    coherence = cm.get_coherence()  # get coherence value
    
    # QUERY ORIGINAL #######
    
    # query topics sorted by score
    initial_query_topics = model.get_document_topics(dictionary.doc2bow(query))
    query_topics = list(sorted(initial_query_topics, key=lambda x: x[1], reverse=True))

    # Get documents with highest probability for each topic. Source: https://stackoverflow.com/a/56814624

    # matrix with the query topics as columns and docs as rows, value is the proba
    topic_matrix = pd.DataFrame(columns=[x[0] for x in query_topics])

    # Loop over all the documents to group the probability of each topic
    for docID in range(len(corpus)):
        topic_matrix.loc[len(topic_matrix)] = 0  # fill with zeros
        topic_vector = OrderedDict(model[corpus[docID]])  # convert list of tuples to OrderedDict to go faster
        for topicID, proba in query_topics:  # only the query topics are relevant
            topic_matrix.at[docID, topicID] = proba * topic_vector.get(topicID, 0)

    # sum probas
    # ids dos docs
    docs = np.array((range(len(corpus))))
    # soma de todas as colunas p/ cada documento
    topic_probas = topic_matrix.sum(axis=1)
    # sorteia da maior p/ a menor proba
    ids = docs[np.argsort(topic_probas[docs])[::-1]]
    print(ids)
    # top_docs = topic_probas[ids]
    
    # todo: calcular a relevancia de cada palavra para a query
    # proba da palavra w * (proba da palavra 1 da query * proba da palavra 2 da query...)
    # ou obter todas as palavras que a relevância seja > 0
    query_per_word_topics = model.get_document_topics(dictionary.doc2bow(query), per_word_topics=True)
    query_per_word_topics = query_per_word_topics[2]
    query_per_word_topics = [OrderedDict(probas) for word_id, probas in query_per_word_topics]
    doc_ids_to_extend = ids[:N]
    new_words_ids = []

    # Loop over all the documents to get the new query words
    for docID in doc_ids_to_extend:
        per_word_topics = model.get_document_topics(corpus[docID], per_word_topics=True)
        per_word_topics = per_word_topics[2]
        
        for wordID, topics_probas in per_word_topics:
            for topicID, proba in topics_probas:
                result = proba
                for d in query_per_word_topics:
                    result *= d.get(topicID, 0)
                
                if result > 0:
                    new_words_ids.append(wordID)
    
    new_words_ids = list(set(new_words_ids))
    new_query = query + [dictionary[wordID] for wordID in new_words_ids]
    
    # QUERY EXTENDIDA #######
    
    # query topics sorted by score
    initial_query_topics = model.get_document_topics(dictionary.doc2bow(new_query))
    query_topics = list(sorted(initial_query_topics, key=lambda x: x[1], reverse=True))

    # Get documents with highest probability for each topic. Source: https://stackoverflow.com/a/56814624

    # matrix with the query topics as columns and docs as rows, value is the proba
    topic_matrix = pd.DataFrame(columns=[x[0] for x in query_topics])

    # Loop over all the documents to group the probability of each topic
    for docID in range(len(corpus)):
        topic_matrix.loc[len(topic_matrix)] = 0  # fill with zeros
        topic_vector = OrderedDict(model[corpus[docID]])  # convert list of tuples to OrderedDict to go faster
        for topicID, proba in query_topics:  # only the query topics are relevant
            topic_matrix.at[docID, topicID] = proba * topic_vector.get(topicID, 0)

    # sum probas
    # ids dos docs
    docs = np.array((range(len(corpus))))
    # soma de todas as colunas p/ cada documento
    topic_probas = topic_matrix.sum(axis=1)
    # sorteia da maior p/ a menor proba
    ids = docs[np.argsort(topic_probas[docs])[::-1]]
    print(ids)
    
    # relvant counter
    num_relevant = 0

    # quantos documentos tenho que revisar até achar o primeiro relevante
    counter = 0
    f_distance = 0
    top_10_true = 0
    top_20_true = 0

    for idx, doc in enumerate(ids):
        counter += 1
        doc_data = df.iloc[doc, :]
        if doc_data['label'] == 'yes':
            num_relevant += 1
            if idx < 10:
                top_10_true += 1
            if idx < 20:
                top_20_true += 1
            if not f_distance:
                f_distance = counter
            counter = 0

    return len(initial_query_topics), coherence, len(ids), len(new_query), num_relevant, f_distance, top_10_true, top_20_true<jupyter_output><empty_output><jupyter_text>Combinações de teste<jupyter_code># Dados de teste: DBs, número de tópicos e sementes do random state,
DATABASES = {'Kitchenham': 'Literature review', 'Radjenovic': 'Defect prediction metrics', 'Hall': 'Defect prediction', 'Wahono': 'Defect prediction'}
NUM_TOPICS_FOR_TEST = range(10, 201, 10)

# dicionário onde serão armazenados os resultados
results = {
    'database': [],                 # nome da base
    'query': [],                    # query
    'num_topics': [],               # número de tópicos utilizado
    'qry_topics': [],               # número de tópicos retornado para a query
    'num_documents': [],            # número total de documentos
    'rand_seed': [],                # semente do random do modelo LDA
    'coherence': [],                # coerência dos tópicos com a coleção
    'ext_query_size': [],           # tamanho da query estendida
    'num_relevant': [],             # número de documentos relevantes encontrados no TOP 10
    'f_distance': [],               # distância até o primeiro relevante
    'top_10_true': [],              # documentos relevantes no top 10
    'top_20_true': [],              # documentos relevantes no top 20
}<jupyter_output><empty_output><jupyter_text>Tests<jupyter_code>print(','.join(['"{}"'.format(key) for key in results.keys()]))

for dbname, query in DATABASES.items():
    path = 'https://github.com/fastread/src/raw/master/workspace/data/{}.csv'.format(dbname)

    #Carregando dataset
    df = pd.read_csv(path, header=0, encoding='latin-1')

    N_DOCS = df.shape[0]  # ilimited n_docs

    base_docs = df['Document Title'] + ' ' + df['Abstract']

    # put de query in the text processing
    base_docs = base_docs.append(pd.Series([query], index=[len(base_docs)]))
    
    corpus, dictionary, docs, query = get_corpus_for_docs(base_docs) # query tem o mesmo preprocessamento da coleção

    for rand_seed in random_numbers:
        for num_topics in NUM_TOPICS_FOR_TEST:
            qry_topics, coherence, num_documents, len_qry, num_relevant, f_distance, top_10_true, top_20_true = get_top_documents(df, corpus, dictionary, query, num_topics, rand_seed)
            values = [dbname, ' '.join(query), str(num_topics), str(qry_topics), str(num_documents), str(rand_seed), str(coherence).replace('.', ','), str(len_qry), str(num_relevant), str(f_distance), str(top_10_true), str(top_20_true)]
            print(','.join(['"{}"'.format(value) for value in values]))
            results['database'].append(dbname)
            results['query'].append(' '.join(query))
            results['num_topics'].append(num_topics)
            results['qry_topics'].append(qry_topics)
            results['num_documents'].append(num_documents)
            results['rand_seed'].append(rand_seed)
            results['coherence'].append(coherence)
            results['ext_query_size'].append(len_qry)
            results['num_relevant'].append(num_relevant)
            results['f_distance'].append(f_distance)
            results['top_10_true'].append(top_10_true)
            results['top_20_true'].append(top_20_true)
df_results = pd.DataFrame(results)
df_results.to_csv(os.path.join(os.getcwd(), "resultados_lda_mallet_v2.csv"))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1034078539730386, 'fraction_numerical': 0.01850456334254375, 'mean_word_length': 3.211565585331453, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '215916', 'license': 'MIT', 'n_tokens_mistral': 4143, 'n_tokens_neox': 3873, 'n_words': 1034, 'path': 'notebook/lda_v2.ipynb', 'repo_name': 'brunoprograma/src', 'size': '11943'}