<commit_before>import os, sys
sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir))

import time
import re
import urllib
import urllib2
import logging
import urlparse
import requests
from bs4 import BeautifulSoup
from datetime import datetime
import traceback

from basecrawler import BaseCrawler
from library.models import *

## =====================================================================
## LOGGING CONFIGURATION
## =====================================================================

LOG = logging.getLogger(__name__)
LOG_handler = logging.StreamHandler()
LOG_formatter = logging.Formatter(fmt='%(asctime)s [%(filename)s:%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s',
                                  datefmt='%m-%d-%Y %H:%M:%S')
LOG_handler.setFormatter(LOG_formatter)
LOG.addHandler(LOG_handler)
LOG.setLevel(logging.INFO)

## =====================================================================
## DRUPAL CONFIGURATION
## =====================================================================

## =====================================================================
## DRUPAL CRAWLER
## =====================================================================
class DrupalCrawler(BaseCrawler):
    def __init__(self, crawlerStatus):
        BaseCrawler.__init__(self, crawlerStatus)
    ## DEF

    def load_url(self, url):
        LOG.info("Retrieving data from %s" % url)
        request = urllib2.Request(url)
        response = urllib2.urlopen(request)
        return response
    ## DEF

    def get_api_data(self, name):
        data = {}
        data['url'] = self.crawlerStatus.source.get_url(name)
        response = requests.get(data['url'])
        soup = BeautifulSoup(response.text, "lxml")
        data['time'] = soup.find('time').attrs['datetime']
        return data

    def add_repository(self, name, setup_scripts):
        if Repository.objects.filter(name=name, source=self.crawlerStatus.source).exists():
            LOG.info("Repository '%s' already exists" % name)
        else:
            try:
                api_data = self.get_api_data(name)
            except:
                traceback.print_exc()
                raise Exception('Not Found')

            # Create the new repository
            repo = Repository()
            repo.name = name
            repo.source = self.crawlerStatus.source
            repo.project_type = self.crawlerStatus.project_type
            repo.last_attempt = None<commit_msg>add a virtual user name drupal for repos crawled from Drupal official webiste<commit_after>import os, sys
sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir))

import time
import re
import urllib
import urllib2
import logging
import urlparse
import requests
from bs4 import BeautifulSoup
from datetime import datetime
import traceback

from basecrawler import BaseCrawler
from library.models import *

## =====================================================================
## LOGGING CONFIGURATION
## =====================================================================

LOG = logging.getLogger(__name__)
LOG_handler = logging.StreamHandler()
LOG_formatter = logging.Formatter(fmt='%(asctime)s [%(filename)s:%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s',
                                  datefmt='%m-%d-%Y %H:%M:%S')
LOG_handler.setFormatter(LOG_formatter)
LOG.addHandler(LOG_handler)
LOG.setLevel(logging.INFO)

## =====================================================================
## DRUPAL CONFIGURATION
## =====================================================================

## =====================================================================
## DRUPAL CRAWLER
## =====================================================================
class DrupalCrawler(BaseCrawler):
    def __init__(self, crawlerStatus):
        BaseCrawler.__init__(self, crawlerStatus)
    ## DEF

    def load_url(self, url):
        LOG.info("Retrieving data from %s" % url)
        request = urllib2.Request(url)
        response = urllib2.urlopen(request)
        return response
    ## DEF

    def get_api_data(self, name):
        data = {}
        data['url'] = self.crawlerStatus.source.get_url(name)
        response = requests.get(data['url'])
        soup = BeautifulSoup(response.text, "lxml")
        data['time'] = soup.find('time').attrs['datetime']
        return data

    def add_repository(self, name, setup_scripts):
        if Repository.objects.filter(name=name, source=self.crawlerStatus.source).exists():
            LOG.info("Repository '%s' already exists" % name)
        else:
            try:
                api_data = self.get_api_data(name)
            except:
                traceback.print_exc()
                raise Exception('Not Found')

            # Create the new repository
            repo = Repository()
            repo.name = 'drupal/' + name
            repo.source = self.crawlerStatus.source
            repo.project_type = self.crawlerStatus.project_type
            repo.last_attempt = None
{'fraction_non_alphanumeric': 0.25992063492063494, 'fraction_numerical': 0.002777777777777778, 'mean_word_length': 3.6078610603290677, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4798378', 'n_tokens_mistral': 1376, 'n_tokens_neox': 1298, 'n_words': 270}