<commit_before>import collections
from urllib import urlencode
from urlparse import parse_qs
from urlparse import urlsplit
from urlparse import urlunsplit

from client.constants import FieldKeyword
from metadata import Metadata


class DropboxMetadata(Metadata):

    def fetch_site_data(self, sanitized_url, status_code):
        return self.generic_fetch_content(sanitized_url, status_code)

    def get_download_url(self, url):
        scheme, netloc, path, query_string, fragment = urlsplit(url)
        query_params = parse_qs(query_string)

        query_params["dl"] = "1"
        new_query_string = urlencode(query_params, doseq=True)

        return urlunsplit((scheme, netloc, path, new_query_string, fragment))

    def parse_content(self, response):
        self.generic_parse_content(response)

        file_list = collections.OrderedDict()
        file_list[FieldKeyword.COUNT] = 1
        file_list[FieldKeyword.DATA] = [
            {
                FieldKeyword.URL: self.get_download_url(response.request_url),
                FieldKeyword.TYPE: None
            }]

        self.prop_map[FieldKeyword.FILES] = file_list<commit_msg>fix dropbox 404 page erorr this is a huge hack, hitting a 404 page on dropbox currently returns 200 so we manually check to see if it's errored by checking the title<commit_after>import collections
from urllib import urlencode
from urlparse import parse_qs
from urlparse import urlsplit
from urlparse import urlunsplit

from bs4 import BeautifulSoup

from client.constants import FieldKeyword
from client.constants import StatusCode
from metadata import Metadata


class DropboxMetadata(Metadata):

    def fetch_site_data(self, sanitized_url, status_code):
        return self.generic_fetch_content(sanitized_url, status_code)

    def get_download_url(self, url):
        scheme, netloc, path, query_string, fragment = urlsplit(url)
        query_params = parse_qs(query_string)

        query_params["dl"] = "1"
        new_query_string = urlencode(query_params, doseq=True)

        return urlunsplit((scheme, netloc, path, new_query_string, fragment))

    def parse_content(self, response):
        soup = BeautifulSoup(response.content)
        title = self.get_title(soup)

        # TODO: currently unsure of how to tell if you have landed on a page that has 404ed since dropbox always returns 200 status
        # the below is a hack to get around this problem, will need to clean this up once we've figured out how to resolve this issue
        if title == 'Dropbox - Error':
            self.prop_map[FieldKeyword.STATUS] = StatusCode.NOT_FOUND
            self.prop_map[FieldKeyword.ERROR_MSG] = StatusCode.get_status_message(StatusCode.NOT_FOUND)

            favicon_url = self.get_favicon_url(soup)
            desc = self.get_desc(soup)

            self.prop_map[FieldKeyword.TITLE] = title
            self.prop_map[FieldKeyword.FAVICON] = favicon_url
            self.prop_map[FieldKeyword.DESC] = desc

        else:
            self.generic_parse_content(response)

            file_list = collections.OrderedDict()
            file_list[FieldKeyword.COUNT] = 1
            file_list[FieldKeyword.DATA] = [
                {
                    FieldKeyword.URL: self.get_download_url(response.sanitized_url),
                    FieldKeyword.TYPE: None
                }]

            self.prop_map[FieldKeyword.FILES] = file_list
{'fraction_non_alphanumeric': 0.06903642773207991, 'fraction_numerical': 0.005875440658049354, 'mean_word_length': 3.0729665071770333, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '345374', 'n_tokens_mistral': 989, 'n_tokens_neox': 958, 'n_words': 253}