<commit_before># This program rearranges raw Egderyders data and builds two lists of dicts, userlist and ciommentslist, containing 
# of the data needed to buildm graphs. These objects are then saved into files.
import os, sys
import json
import csv
from datetime import datetime
import time
import networkx as nx
import logging

from edgesense.utils.logger_initializer import initialize_logger
import edgesense.utils as eu
import edgesense.network as en

def build(allusers, allnodes, allcomments, timestamp, node_title_field='uid', timestep_size=60*60*24*7, timestep_window=1, timestep_count=None, admin_roles=set()):
    # this is the network object
    # going forward it should be read from a serialized format to handle caching
    network = {}

    # Add some file metadata
    network['meta'] = {}
    # Timestamp of the file generation (to show in the dashboard)
    network['meta']['generated'] = int(timestamp.strftime("%s"))

    # build a mapping of nodes (users) keyed on their id
    nodes_map = {}
    for user in allusers:
        if not nodes_map.has_key(user['uid']):
            user_data = {}
            user_data['id'] = user['uid']
            if user.has_key(node_title_field):
                user_data['name'] = user[node_title_field]
            else:
                user_data['name'] = "User %(uid)s" % user
            if user.has_key('link') and user['link']!='':
                user_data['link'] = user['link']    
            # timestamps
            user_data['created_ts'] = int(user['created'])
            user_data['created_on'] = datetime.fromtimestamp(user_data['created_ts']).date().isoformat()
            # team membership
            user_data['team'] = eu.extract.is_team(user, admin_roles)
            user_data['team_ts'] = user_data['created_ts'] if user_data['team'] else None # this would be different if we'd start from previous data dump
            user_data['team_on'] = user_data['created_on'] if user_data['team'] else None # this would be different if we'd start from previous data dump
            user_data['active'] = False
            nodes_map[user['uid']] = user_data
        else:
            print "User %(uid)s was alredy added to the map (??)" % user

    network['nodes'] = nodes_map

    logging.info("nodes collected")  

    # build a mapping of posts keyed by their post id (nid)
    posts_map = {}
    for post in allnodes:
        if not posts_map.has_key(post['nid']):
            post_data = {}
            post_data['id'] = post['nid']
            # timestamps
            post_data['created_ts'] = int(post['created'])
            post_data['created_on'] = datetime.fromtimestamp(post_data['created_ts']).date().isoformat()
            # author (& team membership)
            if post.has_key('uid') and nodes_map.has_key(post['uid']):
                post_data['author_id'] = post['uid']
                author = nodes_map[post_data['author_id']]
                post_data['team'] = author['team'] and author['team_ts'] <= post_data['created_ts']
            else:<commit_msg>adds also isolated nodes to the network<commit_after># This program rearranges raw Egderyders data and builds two lists of dicts, userlist and ciommentslist, containing 
# of the data needed to buildm graphs. These objects are then saved into files.
import os, sys
import json
import csv
from datetime import datetime
import time
import networkx as nx
import logging

from edgesense.utils.logger_initializer import initialize_logger
import edgesense.utils as eu
import edgesense.network as en

def build(allusers, allnodes, allcomments, timestamp, node_title_field='uid', timestep_size=60*60*24*7, timestep_window=1, timestep_count=None, admin_roles=set()):
    # this is the network object
    # going forward it should be read from a serialized format to handle caching
    network = {}

    # Add some file metadata
    network['meta'] = {}
    # Timestamp of the file generation (to show in the dashboard)
    network['meta']['generated'] = int(timestamp.strftime("%s"))

    # build a mapping of nodes (users) keyed on their id
    nodes_map = {}
    for user in allusers:
        if not nodes_map.has_key(user['uid']):
            user_data = {}
            user_data['id'] = user['uid']
            if user.has_key(node_title_field):
                user_data['name'] = user[node_title_field]
            else:
                user_data['name'] = "User %(uid)s" % user
            if user.has_key('link') and user['link']!='':
                user_data['link'] = user['link']    
            # timestamps
            user_data['created_ts'] = int(user['created'])
            user_data['created_on'] = datetime.fromtimestamp(user_data['created_ts']).date().isoformat()
            # team membership
            user_data['team'] = eu.extract.is_team(user, admin_roles)
            user_data['team_ts'] = user_data['created_ts'] if user_data['team'] else None # this would be different if we'd start from previous data dump
            user_data['team_on'] = user_data['created_on'] if user_data['team'] else None # this would be different if we'd start from previous data dump
            user_data['active'] = False
            nodes_map[user['uid']] = user_data
        else:
            print "User %(uid)s was alredy added to the map (??)" % user

    network['nodes'] = nodes_map

    logging.info("nodes collected")  

    # build a mapping of posts keyed by their post id (nid)
    posts_map = {}
    for post in allnodes:
        if not posts_map.has_key(post['nid']):
            post_data = {}
            post_data['id'] = post['nid']
            # timestamps
            post_data['created_ts'] = int(post['created'])
            post_data['created_on'] = datetime.fromtimestamp(post_data['created_ts']).date().isoformat()
            # author (& team membership)
            if post.has_key('uid') and nodes_map.has_key(post['uid']):
                post_data['author_id'] = post['uid']
                author = nodes_map[post_data['author_id']]
                post_data['team'] = author['team'] and author['team_ts'] <= post_data['created_ts']
                nodes_map[post_data['author_id']]['active'] = True
            else:
{'fraction_non_alphanumeric': 0.11680360882874175, 'fraction_numerical': 0.0025777348155308524, 'mean_word_length': 2.8606965174129355, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1272036', 'n_tokens_mistral': 1814, 'n_tokens_neox': 1770, 'n_words': 509}