<commit_before># This file is part of PyBOSSA.
#
# PyBOSSA is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyBOSSA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBOSSA.  If not, see <http://www.gnu.org/licenses/>.

from StringIO import StringIO
from flaskext.wtf import Form, TextField, validators
from flaskext.babel import lazy_gettext
from pybossa.util import unicode_csv_reader
import json
import requests


importers = []

def register_importer(cls):
    importers.append(cls)
    return cls

def enabled_importers(enabled_importer_names=None):
    if enabled_importer_names is None:
        return importers
    check = lambda i: i.template_id in enabled_importer_names
    return filter(check, importers)


class BulkImportException(Exception):
    pass


googledocs_urls = {
    'spreadsheet': None,
    'image': "https://docs.google.com/spreadsheet/ccc"
             "?key=<KEY>"
             "&usp=sharing",
    'sound': "https://docs.google.com/spreadsheet/ccc"
             "?key=<KEY>"
             "&usp=sharing",
    'map': "https://docs.google.com/spreadsheet/ccc"
           "?key=<KEY>"
           "&usp=sharing",
    'pdf': "https://docs.google.com/spreadsheet/ccc"
           "?key=<KEY>"
           "&usp=sharing"}


class BulkTaskImportForm(Form):
    template_id = None
    form_id = None
    form_detector = None
    tasks = None
    get_data_url = None

    def __init__(self, *args, **kwargs):
        super(BulkTaskImportForm, self).__init__(*args, **kwargs)

    def import_csv_tasks(self, app, csvreader):
        headers = []
        fields = set(['state', 'quorum', 'calibration', 'priority_0',
                      'n_answers'])
        field_header_index = []

        for row in csvreader:
            print row
            if not headers:
                headers = row
                if len(headers) != len(set(headers)):
                    msg = lazy_gettext('The file you uploaded has '
                                       'two headers with the same name.')
                    raise BulkImportException(msg)
                field_headers = set(headers) & fields
                for field in field_headers:
                    field_header_index.append(headers.index(field))
            else:
                task_data = {"info": {}}
                for idx, cell in enumerate(row):
                    if idx in field_header_index:
                        task_data[headers[idx]] = cell
                    else:
                        task_data["info"][headers[idx]] = cell
                yield task_data

    def get_csv_data_from_request(self, app, r):
        if r.status_code == 403:
            msg = "Oops! It looks like you don't have permission to access" \
                " that file"
            raise BulkImportException(lazy_gettext(msg), 'error')
        if ((not 'text/plain' in r.headers['content-type']) and
            (not 'text/csv' in r.headers['content-type'])):
            msg = lazy_gettext(
                "Oops! That file doesn't look like the right file.")
            raise BulkImportException(msg, 'error')

        csvcontent = StringIO(r.text)
        csvreader = unicode_csv_reader(csvcontent)
        return self.import_csv_tasks(app, csvreader)

    @property
    def variants(self):
        return [self.template_id]

@register_importer
class BulkTaskCSVImportForm(BulkTaskImportForm):
    msg_required = lazy_gettext("You must provide a URL")
    msg_url = lazy_gettext("Oops! That's not a valid URL. "
                           "You must provide a valid URL")
    csv_url = TextField(lazy_gettext('URL'),
                        [validators.Required(message=msg_required),
                         validators.URL(message=msg_url)])
    template_id = "csv"
    form_id = "csvform"
    form_detector = "csv_url"

    def get_data_url(self, form):
        return form.csv_url.data

    def tasks(self, app, form):
        dataurl = self.get_data_url(form)
        r = requests.get(dataurl)
        return self.get_csv_data_from_request(app, r)


@register_importer
class BulkTaskGDImportForm(BulkTaskImportForm):
    msg_required = lazy_gettext("You must provide a URL")
    msg_url = lazy_gettext("Oops! That's not a valid URL. "
                           "You must provide a valid URL")
    googledocs_url = TextField(lazy_gettext('URL'),
                               [validators.Required(message=msg_required),
                                   validators.URL(message=msg_url)])
    template_id = "gdocs"
    form_id = "gdform"
    form_detector = "googledocs_url"

    def get_data_url(self, form):
        return ''.join([form.googledocs_url.data, '&output=csv'])

    def tasks(self, app, form):
        dataurl = self.get_data_url(form)
        r = requests.get(dataurl)
        return self.get_csv_data_from_request(app, r)

    @property
    def variants(self):
        return [("-".join([self.template_id, mode]))
                for mode in googledocs_urls.keys()]


@register_importer
class BulkTaskEpiCollectPlusImportForm(BulkTaskImportForm):
    msg_required = lazy_gettext("You must provide an EpiCollect Plus "
                                "project name")
    msg_form_required = lazy_gettext("You must provide a Form name "
                                     "for the project")
    epicollect_project = TextField(lazy_gettext('Project Name'),
                                   [validators.Required(message=msg_required)])
    epicollect_form = TextField(lazy_gettext('Form name'),
                                [validators.Required(message=msg_required)])
    template_id = "epicollect"
    form_id = "epiform"
    form_detector = "epicollect_project"

    def import_epicollect_tasks(self, app, data):
        for d in data:
            yield {"info": d}

    def get_data_url(self, form):
        return 'http://plus.epicollect.net/%s/%s.json' % \
            (form.epicollect_project.data, form.epicollect_form.data)

    def get_epicollect_data_from_request(self, app, r):
        if r.status_code == 403:
            msg = "Oops! It looks like you don't have permission to access" \
                " the EpiCollect Plus project"
            raise BulkImportException(lazy_gettext(msg), 'error')
        if not 'application/json' in r.headers['content-type']:
            msg = "Oops! That project and form do not look like the right one."
            raise BulkImportException(lazy_gettext(msg), 'error')
        return self.import_epicollect_tasks(app, json.loads(r.text))

    def tasks(self, app, form):
        dataurl = self.get_data_url(form)
        r = requests.get(dataurl)
        return self.get_epicollect_data_from_request(app, r)
<commit_msg>remove extraneous references to app from internals of importers<commit_after># This file is part of PyBOSSA.
#
# PyBOSSA is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyBOSSA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBOSSA.  If not, see <http://www.gnu.org/licenses/>.

from StringIO import StringIO
from flaskext.wtf import Form, TextField, validators
from flaskext.babel import lazy_gettext
from pybossa.util import unicode_csv_reader
import json
import requests


importers = []

def register_importer(cls):
    importers.append(cls)
    return cls

def enabled_importers(enabled_importer_names=None):
    if enabled_importer_names is None:
        return importers
    check = lambda i: i.template_id in enabled_importer_names
    return filter(check, importers)


class BulkImportException(Exception):
    pass


googledocs_urls = {
    'spreadsheet': None,
    'image': "https://docs.google.com/spreadsheet/ccc"
             "?key=<KEY>"
             "&usp=sharing",
    'sound': "https://docs.google.com/spreadsheet/ccc"
             "?key=<KEY>"
             "&usp=sharing",
    'map': "https://docs.google.com/spreadsheet/ccc"
           "?key=<KEY>"
           "&usp=sharing",
    'pdf': "https://docs.google.com/spreadsheet/ccc"
           "?key=<KEY>"
           "&usp=sharing"}


class BulkTaskImportForm(Form):
    template_id = None
    form_id = None
    form_detector = None
    tasks = None
    get_data_url = None

    def __init__(self, *args, **kwargs):
        super(BulkTaskImportForm, self).__init__(*args, **kwargs)

    def import_csv_tasks(self, csvreader):
        headers = []
        fields = set(['state', 'quorum', 'calibration', 'priority_0',
                      'n_answers'])
        field_header_index = []

        for row in csvreader:
            print row
            if not headers:
                headers = row
                if len(headers) != len(set(headers)):
                    msg = lazy_gettext('The file you uploaded has '
                                       'two headers with the same name.')
                    raise BulkImportException(msg)
                field_headers = set(headers) & fields
                for field in field_headers:
                    field_header_index.append(headers.index(field))
            else:
                task_data = {"info": {}}
                for idx, cell in enumerate(row):
                    if idx in field_header_index:
                        task_data[headers[idx]] = cell
                    else:
                        task_data["info"][headers[idx]] = cell
                yield task_data

    def get_csv_data_from_request(self, r):
        if r.status_code == 403:
            msg = "Oops! It looks like you don't have permission to access" \
                " that file"
            raise BulkImportException(lazy_gettext(msg), 'error')
        if ((not 'text/plain' in r.headers['content-type']) and
            (not 'text/csv' in r.headers['content-type'])):
            msg = lazy_gettext(
                "Oops! That file doesn't look like the right file.")
            raise BulkImportException(msg, 'error')

        csvcontent = StringIO(r.text)
        csvreader = unicode_csv_reader(csvcontent)
        return self.import_csv_tasks(csvreader)

    @property
    def variants(self):
        return [self.template_id]

@register_importer
class BulkTaskCSVImportForm(BulkTaskImportForm):
    msg_required = lazy_gettext("You must provide a URL")
    msg_url = lazy_gettext("Oops! That's not a valid URL. "
                           "You must provide a valid URL")
    csv_url = TextField(lazy_gettext('URL'),
                        [validators.Required(message=msg_required),
                         validators.URL(message=msg_url)])
    template_id = "csv"
    form_id = "csvform"
    form_detector = "csv_url"

    def get_data_url(self, form):
        return form.csv_url.data

    def tasks(self, app, form):
        dataurl = self.get_data_url(form)
        r = requests.get(dataurl)
        return self.get_csv_data_from_request(r)


@register_importer
class BulkTaskGDImportForm(BulkTaskImportForm):
    msg_required = lazy_gettext("You must provide a URL")
    msg_url = lazy_gettext("Oops! That's not a valid URL. "
                           "You must provide a valid URL")
    googledocs_url = TextField(lazy_gettext('URL'),
                               [validators.Required(message=msg_required),
                                   validators.URL(message=msg_url)])
    template_id = "gdocs"
    form_id = "gdform"
    form_detector = "googledocs_url"

    def get_data_url(self, form):
        return ''.join([form.googledocs_url.data, '&output=csv'])

    def tasks(self, app, form):
        dataurl = self.get_data_url(form)
        r = requests.get(dataurl)
        return self.get_csv_data_from_request(r)

    @property
    def variants(self):
        return [("-".join([self.template_id, mode]))
                for mode in googledocs_urls.keys()]


@register_importer
class BulkTaskEpiCollectPlusImportForm(BulkTaskImportForm):
    msg_required = lazy_gettext("You must provide an EpiCollect Plus "
                                "project name")
    msg_form_required = lazy_gettext("You must provide a Form name "
                                     "for the project")
    epicollect_project = TextField(lazy_gettext('Project Name'),
                                   [validators.Required(message=msg_required)])
    epicollect_form = TextField(lazy_gettext('Form name'),
                                [validators.Required(message=msg_required)])
    template_id = "epicollect"
    form_id = "epiform"
    form_detector = "epicollect_project"

    def import_epicollect_tasks(self, data):
        for d in data:
            yield {"info": d}

    def get_data_url(self, form):
        return 'http://plus.epicollect.net/%s/%s.json' % \
            (form.epicollect_project.data, form.epicollect_form.data)

    def get_epicollect_data_from_request(self, r):
        if r.status_code == 403:
            msg = "Oops! It looks like you don't have permission to access" \
                " the EpiCollect Plus project"
            raise BulkImportException(lazy_gettext(msg), 'error')
        if not 'application/json' in r.headers['content-type']:
            msg = "Oops! That project and form do not look like the right one."
            raise BulkImportException(lazy_gettext(msg), 'error')
        return self.import_epicollect_tasks(json.loads(r.text))

    def tasks(self, app, form):
        dataurl = self.get_data_url(form)
        r = requests.get(dataurl)
        return self.get_epicollect_data_from_request(r)

{'fraction_non_alphanumeric': 0.09722513454952121, 'fraction_numerical': 0.0011183336828126092, 'mean_word_length': 2.3367537313432836, 'pattern_counts': {'":': 4, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 8, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1518126', 'n_tokens_mistral': 4324, 'n_tokens_neox': 4115, 'n_words': 1077}