<commit_before>from dependencywatcher.crawler.detectors import Detector
from dependencywatcher.crawler.utils import VersionUtil
import urllib2, json, logging

logger = logging.getLogger(__name__)

class ClojarsDetector(Detector):
    """ clojars.org registry API based information detector """
    api_url = "https://clojars.org/api/artifacts/%s"

    def __init__(self, manifest):
        self.json = None
        clojars_url = "https://clojars.org/%s" % manifest["name"].replace(":", "/")
        self.web_detector = Detector.create("xpath", {
            "name": manifest["name"],
            "detectors": {
                "updatetime": {
                    "xpath": {
                        "url": clojars_url,
                        "xpath": "//span[@title != '']/@title",
                        "dateFormat": "%a %b %d %H:%M:%S %Z %Y"
                    }
                },
                "url": {
                    "xpath": {
                        "url": clojars_url,
                        "xpath": "//ul[@id='jar-info-bar']//a/@href"
                    }
                }
            }
        })
        super(ClojarsDetector, self).__init__(manifest) 

    def detect(self, what, options, result):
        if self.json is None:
            url = ClojarsDetector.api_url % self.manifest["name"].replace(":", "/")
            logger.debug("Opening URL: %s" % url)
            self.json = json.load(urllib2.urlopen(url))

        try:
            if what == "url":
                if what in self.json:
                    result[what] = self.normalize(what, self.json["homepage"])
                else:
                    self.web_detector.detect(what, self.web_detector.manifest["detectors"][what]["xpath"], result)
            elif what in ["description", "license", "updatetime"]:
                if what in self.json:
                    result[what] = self.normalize(what, self.json[what])
                else:
                    self.web_detector.detect(what, self.web_detector.manifest["detectors"][what]["xpath"], result)
            elif what == "version" or what == "stable_version":
                versions = [v["version"] for v in self.json["recent_versions"]]
                result[what] = VersionUtil.find_latest(versions) if what == "version" else VersionUtil.find_stable(versions)
        except KeyError:<commit_msg>Use clojars URL as homepage if the original can't be found<commit_after>from dependencywatcher.crawler.detectors import Detector
from dependencywatcher.crawler.utils import VersionUtil
import urllib2, json, logging

logger = logging.getLogger(__name__)

class ClojarsDetector(Detector):
    """ clojars.org registry API based information detector """
    api_url = "https://clojars.org/api/artifacts/%s"

    def __init__(self, manifest):
        self.json = None
        self.clojars_url = "https://clojars.org/%s" % manifest["name"].replace(":", "/")
        self.web_detector = Detector.create("xpath", {
            "name": manifest["name"],
            "detectors": {
                "updatetime": {
                    "xpath": {
                        "url": self.clojars_url,
                        "xpath": "//span[@title != '']/@title",
                        "dateFormat": "%a %b %d %H:%M:%S %Z %Y"
                    }
                },
                "url": {
                    "xpath": {
                        "url": self.clojars_url,
                        "xpath": "//ul[@id='jar-info-bar']//a/@href"
                    }
                }
            }
        })
        super(ClojarsDetector, self).__init__(manifest) 

    def detect(self, what, options, result):
        if self.json is None:
            url = ClojarsDetector.api_url % self.manifest["name"].replace(":", "/")
            logger.debug("Opening URL: %s" % url)
            self.json = json.load(urllib2.urlopen(url))

        try:
            if what == "url":
                if what in self.json:
                    result[what] = self.normalize(what, self.json["homepage"])
                else:
                    self.web_detector.detect(what, self.web_detector.manifest["detectors"][what]["xpath"], result)
                if not result[what]:
                    result[what] = self.clojars_url
            elif what in ["description", "license", "updatetime"]:
                if what in self.json:
                    result[what] = self.normalize(what, self.json[what])
                else:
                    self.web_detector.detect(what, self.web_detector.manifest["detectors"][what]["xpath"], result)
            elif what == "version" or what == "stable_version":
                versions = [v["version"] for v in self.json["recent_versions"]]
                result[what] = VersionUtil.find_latest(versions) if what == "version" else VersionUtil.find_stable(versions)
        except KeyError:
{'fraction_non_alphanumeric': 0.13810794171967986, 'fraction_numerical': 0.0008208495793145906, 'mean_word_length': 1.7552289429055963, 'pattern_counts': {'":': 30, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5698113', 'n_tokens_mistral': 1426, 'n_tokens_neox': 1351, 'n_words': 274}