<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = "<NAME> <<EMAIL>>"
__date__ = "24/03/2014"
__version__ = "$Revision: 1.0"

from optparse import OptionParser
from operator import itemgetter
from collections import deque
#from igraph import *
import sys
import time
import os


#*****************************************************************************#
#                                                                             #
#                                Data structure                               #
#                                                                             #
#*****************************************************************************#

class Node(object):
    """
    Define the data structure of a node and its direct neighbours
    """
    neighbours = []
    parent = -1
    name = ''
    num = -1
    abundance = 0
    belongingRoot = -1
    checked = False
    seed = False

    # Initializer
    def __init__(self):
        self.neighbours = []


def create_node():
    """
    Create node objects
    """
    node = Node()
    return node


#*****************************************************************************#
#                                                                             #
#                                   Helpers                                   #
#                                                                             #
#*****************************************************************************#

def option_parse():
    """
    Parse arguments from command line.
    """
    desc = """Break swarms."""

    parser = OptionParser(usage="usage: %prog --data_file filename",
                          description=desc,
                          version="%prog version 1.0")

    parser.add_option("-f", "--fasta_file",
                      metavar="<FILENAME>",
                      action="store",
                      dest="fasta_file",
                      help="set <FILENAME> as fasta file.")

    parser.add_option("-s", "--swarm_file",
                      metavar="<FILENAME>",
                      action="store",
                      dest="swarm_file",
                      help="set <FILENAME> as swarm file.")

    parser.add_option("-d", "--data_file",
                      metavar="<FILENAME>",
                      action="store",
                      dest="data_file",
                      help="set <FILENAME> as data file.")

    parser.add_option("-t", "--threshold",
                      metavar="<VALUE>",
                      type=float,
                      default=100,
                      action="store",
                      dest="threshold",
                      help="set <VALUE> as threshold.")

    parser.add_option("-m", "--manual",
                      default=False,
                      action="store_true",
                      dest="manualCut",
                      help="Activate manual cutting mode.")

    parser.add_option("-p", "--parameters",
                      default=True,
                      action="store_false",
                      dest="parameters",
                      help="Deactivate the use of parameters.")

    (options, args) = parser.parse_args()

    return options.fasta_file, options.swarm_file, options.data_file, \
        options.threshold, options.manualCut, options.parameters


def find_path(graph, start, end, path=[]):
    """
    Find the shortest path between two nodes
    """
    path = path + [start]
    if start == end:
        return path
    for node in graph[start].neighbours:
        if node not in path:
            newpath = find_path(graph, node, end, path)
            if newpath:
                return newpath
    return None


def rewire(G, node, threshold, root=True):
    """
    If belonging root is below threshold, rewire node from breaking
    point's belonging root or from breaking point. Performs
    breadth-first search (BFS) from starting point to find new
    belonging root. Testing which version is better - rewire from root
    seems most promising.
    """
    newParent = -1
    belongingRootAbundance = 0
    starting_point = G[node].belongingRoot if root else node
    queue = deque([starting_point])
    while newParent < 0:
        for neighbour in G[queue.popleft()].neighbours:
            curAbundance = G[G[neighbour].belongingRoot].abundance
            if curAbundance > threshold \
                    and curAbundance > belongingRootAbundance:
                newParent = neighbour
                belongingRootAbundance = curAbundance
            else:
                queue.append(neighbour)
    # G[node].parent = newParent
    G[node].belongingRoot = G[newParent].belongingRoot
    return None


def outputSwarmFile(G, new_swarms, swarm_file):
    """
    Output new swarm file
    """
    tim = time.clock()

    output_file_swarm = os.path.splitext(swarm_file)[0]+"_new.swarm"
    with open(output_file_swarm, 'w') as f:
        for swarm in new_swarms:
            for node in swarm:
                f.write(str(G[node[0]].name)+"_"+str(node[1])+" ")
            f.write("\n")
    f.close()

    print "Time used to make output files:", time.clock()-tim


#*****************************************************************************#
#                                                                             #
#                                  Initializer                                #
#                                                                             #
#*****************************************************************************#
def buildGraph(fasta_file, swarm_file, data_file):
    """
    Set up data structure (graph)
    """
    print "Building data structure"
    tim = time.clock()

    with open(swarm_file, "rU") as swarm_file:
        for line in swarm_file:
            amplicons = [(amplicon.split("_")[0], int(amplicon.split("_")[1]))
                         for amplicon in line.strip().split(" ")]

    amplicon_index = {amplicon[0]: i for (i, amplicon) in enumerate(amplicons)}
    #nodeNames = [name[0] for name in amplicons]
    #abundance = [abund[1] for abund in amplicons]

    # Initialize graph structure
    G = [create_node() for i in range(len(amplicon_index))]
    print "Network size:", len(G)

    # Insert name and abundance for each node
    for i in range(len(G)):
        G[i].name = amplicons[i][0]  # The nodes hashed name
        G[i].abundance = int(amplicons[i][1])  # the node's abundance
        G[i].num = i  # ID in graph

    if data_file:
        # Create list of neighbours
        with open(data_file, "rU") as data_file:
            for line in data_file:
                ampliconA, ampliconB, differences = line.split()
                ampliconA = amplicon_index[ampliconA]
                ampliconB = amplicon_index[ampliconB]
                G[ampliconA].neighbours.append(ampliconB)
                G[ampliconB].neighbours.append(ampliconA)
    #elif fasta_file:
        ### CODE FOR FASTA FILE HERE ###
    else:
        print "ERROR: NO DATA FILE OR FASTA FILE GIVEN"
        sys.exit(0)

    print "Time:", time.clock()-tim<commit_msg>Remove graph initialization<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# from __future__ import print_function as print2

__author__ = "<NAME> <<EMAIL>>"
__date__ = "24/03/2014"
__version__ = "$Revision: 1.0"

from optparse import OptionParser
from operator import itemgetter
from collections import deque
#from igraph import *
import sys
import time
import os


#*****************************************************************************#
#                                                                             #
#                                Data structure                               #
#                                                                             #
#*****************************************************************************#

class Node(object):
    """
    Define the data structure of a node and its direct neighbours
    """
    neighbours = []
    parent = -1
    name = ''
    num = -1
    abundance = 0
    belongingRoot = -1
    checked = False
    seed = False

    # Initializer
    def __init__(self):
        self.neighbours = []


def create_node():
    """
    Create node objects
    """
    node = Node()
    return node


#*****************************************************************************#
#                                                                             #
#                                   Helpers                                   #
#                                                                             #
#*****************************************************************************#

def option_parse():
    """
    Parse arguments from command line.
    """
    desc = """Break swarms."""

    parser = OptionParser(usage="usage: %prog --data_file filename",
                          description=desc,
                          version="%prog version 1.0")

    parser.add_option("-f", "--fasta_file",
                      metavar="<FILENAME>",
                      action="store",
                      dest="fasta_file",
                      help="set <FILENAME> as fasta file.")

    parser.add_option("-s", "--swarm_file",
                      metavar="<FILENAME>",
                      action="store",
                      dest="swarm_file",
                      help="set <FILENAME> as swarm file.")

    parser.add_option("-d", "--data_file",
                      metavar="<FILENAME>",
                      action="store",
                      dest="data_file",
                      help="set <FILENAME> as data file.")

    parser.add_option("-t", "--threshold",
                      metavar="<VALUE>",
                      type=float,
                      default=100,
                      action="store",
                      dest="threshold",
                      help="set <VALUE> as threshold.")

    parser.add_option("-m", "--manual",
                      default=False,
                      action="store_true",
                      dest="manualCut",
                      help="Activate manual cutting mode.")

    parser.add_option("-p", "--parameters",
                      default=True,
                      action="store_false",
                      dest="parameters",
                      help="Deactivate the use of parameters.")

    (options, args) = parser.parse_args()

    return options.fasta_file, options.swarm_file, options.data_file, \
        options.threshold, options.manualCut, options.parameters


def find_path(graph, start, end, path=[]):
    """
    Find the shortest path between two nodes
    """
    path = path + [start]
    if start == end:
        return path
    for node in graph[start].neighbours:
        if node not in path:
            newpath = find_path(graph, node, end, path)
            if newpath:
                return newpath
    return None


def rewire(G, node, threshold, root=True):
    """
    If belonging root is below threshold, rewire node from breaking
    point's belonging root or from breaking point. Performs
    breadth-first search (BFS) from starting point to find new
    belonging root. Testing which version is better - rewire from root
    seems most promising.
    """
    newParent = -1
    belongingRootAbundance = 0
    starting_point = G[node].belongingRoot if root else node
    queue = deque([starting_point])
    while newParent < 0:
        for neighbour in G[queue.popleft()].neighbours:
            curAbundance = G[G[neighbour].belongingRoot].abundance
            if curAbundance > threshold \
                    and curAbundance > belongingRootAbundance:
                newParent = neighbour
                belongingRootAbundance = curAbundance
            else:
                queue.append(neighbour)
    # G[node].parent = newParent
    G[node].belongingRoot = G[newParent].belongingRoot
    return None


def outputSwarmFile(G, new_swarms, swarm_file):
    """
    Output new swarm file
    """

    tim = time.clock()

    output_file_swarm = os.path.splitext(swarm_file)[0] + "_new.swarm"
    with open(output_file_swarm, 'w') as f:
        for swarm in new_swarms:
            for node in swarm:
                # print2(node)
                f.write(str(G[node[0]].name)+"_"+str(node[1])+" ")
            f.write("\n")
    f.close()

    print "Time used to make output files:", time.clock()-tim
    return


#*****************************************************************************#
#                                                                             #
#                                  Initializer                                #
#                                                                             #
#*****************************************************************************#
def buildGraph(fasta_file, swarm_file, data_file):
    """
    Set up data structure (graph)
    """
    print "Building data structure"
    tim = time.clock()

    with open(swarm_file, "rU") as swarm_file:
        for line in swarm_file:
            amplicons = [(amplicon.split("_")[0], int(amplicon.split("_")[1]))
                         for amplicon in line.strip().split(" ")]

    amplicon_index = {amplicon[0]: i for (i, amplicon) in enumerate(amplicons)}

    # Insert name and abundance of each node in the graph
    G = []
    for i in range(len(amplicon_index)):
        G.append(create_node())
        G[i].name = amplicons[i][0]  # The nodes hashed name
        G[i].abundance = int(amplicons[i][1])  # the node's abundance
        G[i].num = i  # ID in graph
    print "Network size:", len(G)

    if data_file:
        # Create list of neighbours
        with open(data_file, "rU") as data_file:
            for line in data_file:
                ampliconA, ampliconB, differences = line.split()
                ampliconA = amplicon_index[ampliconA]
                ampliconB = amplicon_index[ampliconB]
                G[ampliconA].neighbours.append(ampliconB)
                G[ampliconB].neighbours.append(ampliconA)
    #elif fasta_file:
        ### CODE FOR FASTA FILE HERE ###
    else:
        print "ERROR: NO DATA FILE OR FASTA FILE GIVEN"
        sys.exit(0)

    print "Time:", time.clock()-tim
{'fraction_non_alphanumeric': 0.1628619153674833, 'fraction_numerical': 0.004732739420935412, 'mean_word_length': 1.6235165236443307, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 42, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '720153', 'n_tokens_mistral': 3826, 'n_tokens_neox': 3559, 'n_words': 920}