<commit_before>'''
Last edit on 11 dec. 2013

@author: <NAME>
'''
from fr.lille1.maven_data_extraction.scrapers.maven.maven_scraper import MavenScrape
import logging as log
import os
import sys

class Main(object):


    def __init__(self):
        self.scraper = MavenScrape('en')


    '''
    Run scraping and fills the database
    '''
    def run(self):
        self.scraper.setConfig('man', 'jeans', 'Jean', 'Bottom')
        itemList = self.scraper.run();

        log.info('\n-- Scraping DONE ! --')

'''
Main
'''
if __name__ == '__main__':
    log.basicConfig(level = log.DEBUG)

    '''
    if len(sys.argv) < 2:
        print('Usage: ' + sys.argv[0] + ' section subsection')
    else:
        Main('en', sys.argv[1], sys.argv[2]).run()
    '''
    Main().run()

<commit_msg>Configuring of the scraping from the root or from the 'com' package root<commit_after>'''
Last edit on 11 dec. 2013

@author: <NAME>
'''
from fr.lille1.maven_data_extraction.scrapers.maven.maven_scraper import \
    MavenScrape
import logging as log
import os
import sys

class Main(object):

    def __init__(self):
        self.scraper = MavenScrape()

    '''
    Run scraping and .pom files
    '''
    def run(self):
        # We scraped all the artefacts (from the root of MCR)
        # self.scraper.setConfig(47)

        # Only the children of 'com' package
        self.scraper.setConfig(3059406)

        # Go!
        itemList = self.scraper.run();

        log.info('-- Scraping DONE ! --')

'''
Main
'''
if __name__ == '__main__':
    log.basicConfig(level = log.INFO)

    '''
    if len(sys.argv) < 2:
        print('Usage: ' + sys.argv[0] + ' section subsection')
    else:
        Main('en', sys.argv[1], sys.argv[2]).run()
    '''
    Main().run()


{'fraction_non_alphanumeric': 0.15231032515687393, 'fraction_numerical': 0.017683970336565887, 'mean_word_length': 2.6164948453608248, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6033557', 'n_tokens_mistral': 650, 'n_tokens_neox': 600, 'n_words': 150}