<commit_before>import re

from .. import license_matcher
from .base import BaseProcessor

URL_DATA_PATTERN = re.compile(r'(www.)?npm(js)?(.com)?\/(package\/)?([\w\-\_]+)')
JSON_DATA_PATTERN = 'http://registry.npmjs.org/{0.repo}/{0.version}'


class NpmProcessor(BaseProcessor):
    name = 'npm'

    def __init__(self, url=None, repo=None, version=None):
        self.url = url
        self.repo = repo or self.extract_url_data()[4]
        self.version = version or 'latest'

    def extract_url_data(self):
        match = URL_DATA_PATTERN.search(self.url)
        return match.groups()

    @classmethod
    def is_match(cls, url):
        return URL_DATA_PATTERN.match(url)

    def get_license(self):
        data_url = JSON_DATA_PATTERN.format(self)
        repo_info = self.get_file_data(
            data_url,
            parse_json=True
        )

        return repo_info.get('license'), data_url<commit_msg>Update npm url pattern to match npm registry urls as well as npmjs.com ones<commit_after>import re

from .. import license_matcher
from .base import BaseProcessor

URL_DATA_PATTERN = re.compile(
    r'(?:(?:http|https))?(?:\://)?'
    '(?:www)?(?:registry)?\.+npm(?:js)?(?:\.(?:com|org))?\/'
    '(?:package\/)?([\w\-\_]+)'
)
JSON_DATA_PATTERN = 'http://registry.npmjs.org/{0.repo}/{0.version}'


class NpmProcessor(BaseProcessor):
    name = 'npm'

    def __init__(self, url=None, repo=None, version=None):
        self.url = url
        self.repo = repo or self.extract_url_data()[0]
        self.version = version or 'latest'

    def extract_url_data(self):
        match = URL_DATA_PATTERN.search(self.url)
        return match.groups()

    @classmethod
    def is_match(cls, url):
        return URL_DATA_PATTERN.match(url)

    def get_license(self):
        data_url = JSON_DATA_PATTERN.format(self)
        repo_info = self.get_file_data(
            data_url,
            parse_json=True
        )

        return repo_info.get('license'), data_url
{'fraction_non_alphanumeric': 0.14509605662285135, 'fraction_numerical': 0.003033367037411527, 'mean_word_length': 2.911067193675889, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4593238', 'n_tokens_mistral': 684, 'n_tokens_neox': 648, 'n_words': 125}