<commit_before># Copyright (C) 2013  <NAME> <<EMAIL>>
#
# This file is part of Firewose.
#
# Firewose is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.


# Reads a Firehose-related XML file and injects it to a DB

import os, sys
import argparse

import firewose.lib.firehose_orm as fhm
from firewose.lib.hash import idify, uniquify

from xml.etree.ElementTree import ParseError as XmlParseError

metadata = fhm.metadata

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, scoped_session

def _get_engine(url, echo):
    return create_engine(url, echo=echo)

def get_engine_session(url, echo=False):
    """
    Given a database URL, returns an SQLAlchemy engine/session
    """
    engine = _get_engine(url, echo=echo)
    session = scoped_session(sessionmaker(bind=engine, autoflush=True))
    return engine, session

def insert_analysis(session, xml_file):
    """
    Given a file object and a session, creates a Firehose Analysis() object
    and inserts it to the db linked to session
    """
    try:
        analysis = fhm.Analysis.from_xml(xml_file)
    except XmlParseError:
        return # if file is empty for example
    except Exception as e:
        print("ERROR while parsing xml: %s" % e)
        return
    
    #idify:
    try:
        (analysis, analysishash) = idify(analysis)
    except Exception as e:
        print("ERROR while idify Analysis: %s" % e)
        import sys; sys.exit()
    
    # unicity:
    try:
        analysis = uniquify(session, analysis)
    except Exception as e:
        print("ERROR while uniquify Analysis: %s" % e)
        import sys; sys.exit()

    session.merge(analysis)
    session.commit()
    
def read_and_create(url, xml_files, drop=False, echo=False):
    engine, session = get_engine_session(url, echo=echo)
    
    if drop:
        metadata.drop_all(bind=engine) # cleans the table (for debugging)
        metadata.create_all(bind=engine)
    
    for file_ in xml_files:
        try:
            insert_analysis(session, file_)
        except Exception as e:
            print("Error in file %s" % file_)
            print(e)
    
    session.remove()

if __name__ == "__main__":
    
    parser = argparse.ArgumentParser(description="Reads XML from standard\
    input and adds the Firehose objects into the specified database")
    parser.add_argument("db_url", help="URL of the database")
    parser.add_argument("xml_file", help="Path of the XML file", nargs="+")
    parser.add_argument("--drop", help="drops the database before filling",
                        action="store_true")
    parser.add_argument("--verbose", help="outputs SQLAlchemy requests",
                        action="store_true")
    args = parser.parse_args()
    <commit_msg>add: % counter in firewose_fill_db<commit_after># Copyright (C) 2013  <NAME> <<EMAIL>>
#
# This file is part of Firewose.
#
# Firewose is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.


# Reads a Firehose-related XML file and injects it to a DB

import os, sys
import argparse

import firewose.lib.firehose_orm as fhm
from firewose.lib.hash import idify, uniquify

from xml.etree.ElementTree import ParseError as XmlParseError

metadata = fhm.metadata

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, scoped_session

def _get_engine(url, echo):
    return create_engine(url, echo=echo)

def get_engine_session(url, echo=False):
    """
    Given a database URL, returns an SQLAlchemy engine/session
    """
    engine = _get_engine(url, echo=echo)
    session = scoped_session(sessionmaker(bind=engine, autoflush=True))
    return engine, session

def insert_analysis(session, xml_file):
    """
    Given a file object and a session, creates a Firehose Analysis() object
    and inserts it to the db linked to session
    """
    try:
        analysis = fhm.Analysis.from_xml(xml_file)
    except XmlParseError:
        return # if file is empty for example
    except Exception as e:
        print("ERROR while parsing xml: %s" % e)
        return
    
    #idify:
    try:
        (analysis, analysishash) = idify(analysis)
    except Exception as e:
        print("ERROR while idify Analysis: %s" % e)
        import sys; sys.exit()
    
    # unicity:
    try:
        analysis = uniquify(session, analysis)
    except Exception as e:
        print("ERROR while uniquify Analysis: %s" % e)
        import sys; sys.exit()

    session.merge(analysis)
    session.commit()
    
def read_and_create(url, xml_files, drop=False, echo=False):
    engine, session = get_engine_session(url, echo=echo)
    
    if drop:
        metadata.drop_all(bind=engine) # cleans the table (for debugging)
        metadata.create_all(bind=engine)
    
    number_of_files = len(xml_files)
    for (counter, file_) in enumerate(xml_files):
        try:
            insert_analysis(session, file_)
        except Exception as e:
            print("Error in file %s" % file_)
            print(e)
        
        # % counter:
        sys.stdout.write(str(int(float(counter + 1)
                                 / float(number_of_files) * 100)))
        sys.stdout.write(" %")
        sys.stdout.write("\r")
        sys.stdout.flush()
    
    session.remove()

if __name__ == "__main__":
    
    parser = argparse.ArgumentParser(description="Reads XML from standard\
    input and adds the Firehose objects into the specified database")
    parser.add_argument("db_url", help="URL of the database")
    parser.add_argument("xml_file", help="Path of the XML file", nargs="+")
    parser.add_argument("--drop", help="drops the database before filling",
                        action="store_true")
    parser.add_argument("--verbose", help="outputs SQLAlchemy requests",
                        action="store_true")
    args = parser.parse_args()
    
{'fraction_non_alphanumeric': 0.08898426323319027, 'fraction_numerical': 0.0020028612303290413, 'mean_word_length': 3.029394812680115, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 17}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '796839', 'n_tokens_mistral': 2037, 'n_tokens_neox': 1993, 'n_words': 706}