<commit_before><commit_msg>add script for composition and decomposition links<commit_after>require "mongo"
require "unicode"

class String
    def number?
        return self.match(/^\d*$/)
    end

    def sanitize
        self.gsub(/^[^[:alnum:]]*/, '').gsub(/[^[:alnum:]]*$/, '')
    end

    def capitalize_with_dashes
        self.split('-').map { |e| Unicode::capitalize e }.join('-')
    end

    def valid?
        !(self.empty? || self.number?)
    end
end

include Mongo

def words(node)
    node['string'].split.inject([]) do |words, word|
        sanitized = word.sanitize
        words << sanitized if sanitized.valid?
        words
    end
end


mongo = MongoClient.new
db = mongo.db 'graph'
nodes = db.collection 'nodes'
edges = db.collection 'edges'

nodes.find({}, fields: ['string', 'language']).each do |tag_node|

    words = words(tag_node)     

    words.each do |word|

        # insert word node
        word_node = {string: word, language: tag_node['language']}
        existing = nodes.find_one(word_node)
        word_id = existing ? existing['_id'] : nodes.insert(word_node)

        # insert word edges
        edges.insert({source: tag_node['_id'], target: word_id, type: "decomposition"})
        edges.insert({source: word_id, target: tag_node['_id'], type: "composition"})
    end
end
{'fraction_non_alphanumeric': 0.15098634294385432, 'fraction_numerical': 0.0, 'mean_word_length': 2.913946587537092, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4607512', 'n_tokens_mistral': 439, 'n_tokens_neox': 418, 'n_words': 106}