<commit_before>
require 'repo'
require 'user'
require 'lang'
require 'lang_usage'

class Hub

    attr_accessor :repos, :users, :langs, :owners

    def initialize
        @repos = {}
        @users = {}
        @langs = {}
        @owners = {}
    end

    def find_lang(name)
        @langs[name]
    end

    def find_popular_repos_by_lang_for(user)
        
    end
    
    def popular_repos
        return @popular_repos if @popular_repos
        @popular_repos = repos.values.sort{|a,b| a.watchers.size <=> b.watchers.size }.reverse
    end

    def popular_repos_by_forks
        return @popular_forked_repos if @popular_forked_repos
        @popular_forked_repos = repos.values.sort_by{|r| r.forks }.reverse
    end

    def import_files
        import(File.new("repos.txt", 'r'), File.new("lang.txt", "r"), File.new("data.txt", "r"))
    end

    def import(repos, langs, users)
        import_repos_from(repos)
        import_langs_from(langs)
        import_users_from(users)
    end

    def import_repos_from(repos_file)

        $stderr.puts "Parsing repos ..." 

        @repos = {}

        while (line = repos_file.gets)
            repo = Repo.new_repo_from(line)
            @repos[repo.id] = repo
            owner = repo.name.split('/').first;
            @owners[owner] ||= []
            @owners[owner] << repo
            repo.owner = owner
        end

        set_sources
        set_forks

       true
    end

    def import_langs_from(langs_file)

        $stderr.puts "Parsing langs ..." 

        @langs = {}

        while (line = langs_file.gets)
            repo_id, data = line.chomp.split(':')
            repo_id = repo_id.to_i

            repo = @repos[repo_id]

            if repo  
                lang_pairs = data.split(',')
                lang_pairs.each do |pair|
                    name, lines = pair.split(';')

                    lang = create_or_find_lang(name)
                    lang.repos << repo

                    usage = LangUsage.new(lang, lines.to_i)
                    repo.langs << usage
                end
            end
        end

        true
    end

    def import_users_from(file)

        $stderr.puts "Parsing users and setting watchers ..." 

        @users = {}

        while (line = file.gets)
            user_id, repo_id = parse_data_line(line)

            user = @users[user_id] = User.new(user_id)

            if @repos[repo_id]
                user.repos << @repos[repo_id] 
                @repos[repo_id].watchers << user
            end
        end

        true
    end

    private
    
    def parse_data_line(line)
        user_id, repo_id = line.chomp.split(':')

        [user_id.to_i, repo_id.to_i]
    end

    def set_sources

        $stderr.puts "Setting repo sources ..." 

        @repos.each_value do |value|
            if value.source && value.source > 0 && @repos.has_key?(value.source)
                source_id = value.source
                value.source = @repos[value.source]
            end
        end
    end

    def set_forks

        $stderr.puts "Setting forks ..." 

        sources = {}
        
        @repos.each_value do |repo|
            if repo.source && repo.source.is_a?(Repo)
                id = repo.source.id
                sources[id] ||= []
                sources[id] << repo
            end
        end

        sources.each_pair do |source, forks|
            @repos[source].forks = forks if @repos[source]
        end
    end

    def create_or_find_lang(name)
        return @langs[name] if @langs.has_key?(name)

        @langs[name] = Lang.new(name)
    end

end<commit_msg>Fix issue where a new users was created for every entry in data.txt<commit_after>
require 'repo'
require 'user'
require 'lang'
require 'lang_usage'

class Hub

    attr_accessor :repos, :users, :langs, :owners

    def initialize
        @repos = {}
        @users = {}
        @langs = {}
        @owners = {}
    end

    def find_lang(name)
        @langs[name]
    end

    def find_popular_repos_by_lang_for(user)
        
    end
    
    def popular_repos
        return @popular_repos if @popular_repos
        @popular_repos = repos.values.sort{|a,b| a.watchers.size <=> b.watchers.size }.reverse
    end

    def popular_repos_by_forks
        return @popular_forked_repos if @popular_forked_repos
        @popular_forked_repos = repos.values.sort_by{|r| r.forks }.reverse
    end

    def import_files
        import(File.new("repos.txt", 'r'), File.new("lang.txt", "r"), File.new("data.txt", "r"))
    end

    def import(repos, langs, users)
        import_repos_from(repos)
        import_langs_from(langs)
        import_users_from(users)
    end

    def import_repos_from(repos_file)

        $stderr.puts "Parsing repos ..." 

        @repos = {}

        while (line = repos_file.gets)
            repo = Repo.new_repo_from(line)
            @repos[repo.id] = repo
            owner = repo.name.split('/').first;
            @owners[owner] ||= []
            @owners[owner] << repo
            repo.owner = owner
        end

        set_sources
        set_forks

       true
    end

    def import_langs_from(langs_file)

        $stderr.puts "Parsing langs ..." 

        @langs = {}

        while (line = langs_file.gets)
            repo_id, data = line.chomp.split(':')
            repo_id = repo_id.to_i

            repo = @repos[repo_id]

            if repo  
                lang_pairs = data.split(',')
                lang_pairs.each do |pair|
                    name, lines = pair.split(';')

                    lang = create_or_find_lang(name)
                    lang.repos << repo

                    usage = LangUsage.new(lang, lines.to_i)
                    repo.langs << usage
                end
            end
        end

        true
    end

    def import_users_from(file)

        $stderr.puts "Parsing users and setting watchers ..." 

        @users = {}

        while (line = file.gets)
            user_id, repo_id = parse_data_line(line)

            user = create_or_find_user(user_id)

            if @repos[repo_id]
                user.repos << @repos[repo_id] 
                @repos[repo_id].watchers << user
            end
        end

        true
    end

    private
    
    def parse_data_line(line)
        user_id, repo_id = line.chomp.split(':')

        [user_id.to_i, repo_id.to_i]
    end

    def set_sources

        $stderr.puts "Setting repo sources ..." 

        @repos.each_value do |value|
            if value.source && value.source > 0 && @repos.has_key?(value.source)
                source_id = value.source
                value.source = @repos[value.source]
            end
        end
    end

    def set_forks

        $stderr.puts "Setting forks ..." 

        sources = {}
        
        @repos.each_value do |repo|
            if repo.source && repo.source.is_a?(Repo)
                id = repo.source.id
                sources[id] ||= []
                sources[id] << repo
            end
        end

        sources.each_pair do |source, forks|
            @repos[source].forks = forks if @repos[source]
        end
    end

    def create_or_find_lang(name)
        return @langs[name] if @langs.has_key?(name)

        @langs[name] = Lang.new(name)
    end

    def create_or_find_user(uid)
        return @users[uid] if @users.has_key?(uid)

        @users[uid] = User.new(uid)
    end

end
{'fraction_non_alphanumeric': 0.10580940827604798, 'fraction_numerical': 0.0002695781102574471, 'mean_word_length': 1.5772837790899619, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2504713', 'n_tokens_mistral': 2509, 'n_tokens_neox': 2277, 'n_words': 483}