<commit_before># -*- coding: utf-8 -*-

#   Copyright 2014 <NAME>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
"""Main loop for all brand scrapers.

If you want to run particular scrapers (for testing), you can put
their names on the command line (e.g. python scraper.py avon kraft).

It's fine to import from this module inside a scraper
(e.g. from scraper import TM_SYMBOLS)
"""
import json
import logging
import pipes
import re
import shlex
import sys
from collections import defaultdict
from os.path import basename
from os.path import dirname
from os.path import join
from os import environ
from os import listdir
from subprocess import CalledProcessError
from subprocess import PIPE
from subprocess import Popen
from traceback import print_exc

import scraperwiki

import scrapers


log = logging.getLogger('scraper')


def main():
    logging.basicConfig(format='%(name)s: %(message)s',
                        level=logging.INFO)


    if environ.get('MORPH_CAMPAIGNS'):
        campaigns = environ['MORPH_CAMPAIGNS'].split(',')
    else:
        campaigns = sys.argv[1:]

    init_tables()

    failed = False

    # Python campaigns
    for campaign in get_scraper_names():
        if campaigns and campaign not in campaigns:
            continue

        log.info('Launching scraper: {}'.format(campaign))
        try:
            scraper = load_scraper(campaign)

            records = list(scraper.scrape_campaign())
            clear_campaign(campaign)
            save_records(campaign, records)
        except:
            failed = True
            print_exc()

    # Ruby campaigns
    for rb in get_ruby_scraper_paths():
        campaign = basename(rb)[:-3]
        if campaigns and campaign not in campaigns:
            continue

        log.info('Launching ruby scraper: {}'.format(campaign))
        try:
            records = run_ruby_scraper(rb)

            clear_campaign(campaign)
            save_records(campaign, records)
        except CalledProcessError as ex:
            failed = True
            log.warn('ruby scraper exited with status {:d}'.format(
                ex.returncode))
        except:
            failed = True
            print_exc()

    sys.exit(int(failed))


def run_ruby_scraper(rb):
    """Run a Ruby scraper. Return a list of records if successful,
    None if not.
    """
    if 'MORPH_RUBY_BIN' in environ:
        ruby_bin = shlex.split(ruby_bin)
    else:
        ruby_bin = ['ruby']

    cmd = ruby_bin + [rb]
    log.info(' '.join(pipes.quote(arg) for arg in cmd))

    p = Popen(cmd, stdout=PIPE)

    try:
        for i, line in enumerate(p.stdout):
            line = line.rstrip()
            if line:
                try:
                    record = json.loads(line)
                    table = record.pop('table')

                    log.info('parsed {} record: {}'.format(
                        table, guess_entity_name(record)))

                    yield table, record
                except:
                    log.info(repr(line))
                    raise

        p.wait()
        if p.returncode != 0:
            raise CalledProcessError(p.returncode, cmd)
    except:
        log.warn('killing ruby scraper')
        p.terminate()
        p.wait()

        raise


# map from table name to fields used for the primary key (not including
# campaign_id). All key fields are currently TEXT
TABLE_TO_KEY_FIELDS = {<commit_msg>handle company names with accents<commit_after># -*- coding: utf-8 -*-

#   Copyright 2014 <NAME>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
"""Main loop for all brand scrapers.

If you want to run particular scrapers (for testing), you can put
their names on the command line (e.g. python scraper.py avon kraft).

It's fine to import from this module inside a scraper
(e.g. from scraper import TM_SYMBOLS)
"""
import json
import logging
import pipes
import re
import shlex
import sys
from collections import defaultdict
from os.path import basename
from os.path import dirname
from os.path import join
from os import environ
from os import listdir
from subprocess import CalledProcessError
from subprocess import PIPE
from subprocess import Popen
from traceback import print_exc

import scraperwiki

import scrapers


log = logging.getLogger('scraper')


def main():
    logging.basicConfig(format='%(name)s: %(message)s',
                        level=logging.INFO)


    if environ.get('MORPH_CAMPAIGNS'):
        campaigns = environ['MORPH_CAMPAIGNS'].split(',')
    else:
        campaigns = sys.argv[1:]

    init_tables()

    failed = False

    # Python campaigns
    for campaign in get_scraper_names():
        if campaigns and campaign not in campaigns:
            continue

        log.info('Launching scraper: {}'.format(campaign))
        try:
            scraper = load_scraper(campaign)

            records = list(scraper.scrape_campaign())
            clear_campaign(campaign)
            save_records(campaign, records)
        except:
            failed = True
            print_exc()

    # Ruby campaigns
    for rb in get_ruby_scraper_paths():
        campaign = basename(rb)[:-3]
        if campaigns and campaign not in campaigns:
            continue

        log.info('Launching ruby scraper: {}'.format(campaign))
        try:
            records = run_ruby_scraper(rb)

            clear_campaign(campaign)
            save_records(campaign, records)
        except CalledProcessError as ex:
            failed = True
            log.warn('ruby scraper exited with status {:d}'.format(
                ex.returncode))
        except:
            failed = True
            print_exc()

    sys.exit(int(failed))


def run_ruby_scraper(rb):
    """Run a Ruby scraper. Return a list of records if successful,
    None if not.
    """
    if 'MORPH_RUBY_BIN' in environ:
        ruby_bin = shlex.split(ruby_bin)
    else:
        ruby_bin = ['ruby']

    cmd = ruby_bin + [rb]
    log.info(' '.join(pipes.quote(arg) for arg in cmd))

    p = Popen(cmd, stdout=PIPE)

    try:
        for i, line in enumerate(p.stdout):
            line = line.rstrip()
            if line:
                try:
                    record = json.loads(line)
                    table = record.pop('table')

                    log.info(u'parsed {} record: {}'.format(
                        table, guess_entity_name(record)))

                    yield table, record
                except:
                    log.info(repr(line))
                    raise

        p.wait()
        if p.returncode != 0:
            raise CalledProcessError(p.returncode, cmd)
    except:
        log.warn('killing ruby scraper')
        p.terminate()
        p.wait()

        raise


# map from table name to fields used for the primary key (not including
# campaign_id). All key fields are currently TEXT
TABLE_TO_KEY_FIELDS = {
{'fraction_non_alphanumeric': 0.08533333333333333, 'fraction_numerical': 0.0030476190476190477, 'mean_word_length': 2.239819004524887, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7164054', 'n_tokens_mistral': 2290, 'n_tokens_neox': 2159, 'n_words': 748}