<commit_before>from __future__ import print_function
from __future__ import unicode_literals

import codecs
import csv
import os
import re
import sqlite3
import string
import sys
import cStringIO

class Source(object):
    def __init__(self, db):
        """Initialize db cursor."""
        self.db = db
        self.c = db.cursor()
    
    def setup(self, source_path):
        """Setup source in mess.db.
        
        Args:
            source_path: A path to a directory containing source molecules.
            
        """
        self.source = source_path
        # check that the provided source is a real file in the right place
        if not (os.path.isdir(self.source)):
            sys.exit('This source is not a valid directory.')
        source_path_parent = os.path.abspath(os.path.join(self.source + '/..'))
        if not (source_path_parent.split('/')[-1] == 'sources'):
            sys.exit("All sources must reside in the 'sources' directory.")
        source_basename = self.source.strip('/').split('/')[-1]
        source_basename = os.path.basename(source_basename)
        source_sql = os.path.join(os.path.splitext(self.source)[0],
                                  '%s.sql' % source_basename)
        if not (os.path.isfile(source_sql)):
            sys.exit(('All sources must have a corresponding sql file '
                      'in their directory.'))
        # insert/update source in the database
        self.c.executescript(codecs.open(source_sql, encoding='utf-8').read())
        # get source id
        q = ('SELECT source_id, name, dirname, url, url_template, last_update '
             'FROM source WHERE dirname=?')
        source_row = self.c.execute(q, (source_basename,)).fetchone()
        # set attributes
        self.id = source_row.source_id
        self.name = source_row.name
        self.dirname = source_row.dirname
        self.url = source_row.url
        self.url_template = source_row.url_template
        self.last_update = source_row.last_update
    
    def files(self):
        """Returns a list of files in the source directory."""
        return os.listdir(self.source)<commit_msg>source can now be specified by basename or path<commit_after>from __future__ import print_function
from __future__ import unicode_literals

import codecs
import csv
import os
import re
import sqlite3
import string
import sys
import cStringIO

class Source(object):
    def __init__(self, db):
        """Initialize db cursor."""
        self.db = db
        self.c = db.cursor()
    
    def setup(self, source):
        """Setup source in mess.db.
        
        Args:
            source_path: A path to a directory containing source molecules.
            
        """
        if not os.path.isdir(source):
            self.source_dir = os.path.join(os.path.dirname( __file__ ), '../', 
                                       'sources/', source)
            if not os.path.isdir(self.source_dir):
                sys.exit('%s is not a valid source or source directory.' %\
                         source)
        else:
            self.source_dir = source
        source_path_parent = os.path.abspath(os.path.join(self.source_dir, 
                                                          '..'))
        if not (source_path_parent.split('/')[-1] == 'sources'):
            sys.exit("All sources must reside in the 'sources' directory.")
        source_basename = self.source_dir.strip('/').split('/')[-1]
        source_basename = os.path.basename(source_basename)
        source_sql = os.path.join(os.path.splitext(self.source_dir)[0],
                                  '%s.sql' % source_basename)
        if not (os.path.isfile(source_sql)):
            sys.exit(('All sources must have a corresponding sql file '
                      'in their directory.'))
        # insert/update source in the database
        self.c.executescript(codecs.open(source_sql, encoding='utf-8').read())
        # get source id
        q = ('SELECT source_id, name, dirname, url, url_template, last_update '
             'FROM source WHERE dirname=?')
        source_row = self.c.execute(q, (source_basename,)).fetchone()
        # set attributes
        self.id = source_row.source_id
        self.name = source_row.name
        self.dirname = source_row.dirname
        self.url = source_row.url
        self.url_template = source_row.url_template
        self.last_update = source_row.last_update
    
    def files(self):
        """Returns a list of files in the source directory."""
        return os.listdir(self.source_dir)
{'fraction_non_alphanumeric': 0.1063922942206655, 'fraction_numerical': 0.0021891418563922942, 'mean_word_length': 2.076767676767677, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7082563', 'n_tokens_mistral': 1291, 'n_tokens_neox': 1274, 'n_words': 338}