<commit_before># coding: utf-8
import datetime
import time
import flask
import requests

import models

bp = flask.Blueprint('home', __name__)

@bp.route('/ruok', methods=['GET', 'POST'])
def ruok():
    return 'imok'

def cleanname(name):
    for prefix in 'http://', 'https://', '/':
        if name.startswith(prefix):
            name = name[len(prefix):]
    for suffix in '/', '.git':
        if name.endswith(suffix):
            name = name[:-len(suffix)]
    return name

def checkrepo(name):
    ''' return desc '''
    payload = {'action': 'package', 'id': name}
    r = requests.get('http://go-search.org/api', params=payload)
    if not r.text.startswith('{'):
        raise Exception('error find package in go-search: %s' % r.text)

    d = r.json()
    print d
    if d['Name'] and d['Name'] != 'main':
        raise Exception('Repo: [%s] is a go-lib, this platform only support main package' % name)
    return d['Synopsis']

@bp.route('/')
@models.db_session
def home():
    address = flask.request.args.get('address')
    if address:
        reponame = cleanname(address)
        if not models.Repo.get(name=reponame):
            try:
                desc = checkrepo(reponame)<commit_msg>load package in gowalker<commit_after># coding: utf-8
import datetime
import time
import flask
import requests

import models

bp = flask.Blueprint('home', __name__)

@bp.route('/ruok', methods=['GET', 'POST'])
def ruok():
    return 'imok'

def cleanname(name):
    for prefix in 'http://', 'https://', '/':
        if name.startswith(prefix):
            name = name[len(prefix):]
    for suffix in '/', '.git':
        if name.endswith(suffix):
            name = name[:-len(suffix)]
    name = name.replace(':', '/')
    return name

def checkrepo(name):
    ''' return desc '''
    payload = {'pkgname': name}
    requests.get('https://gowalker.org/'+name) # call gowalker to load data
    r = requests.get('https://gowalker.org/api/v1/pkginfo', params=payload)
    d = r.json()
    if not d['id']:
        raise Exception('error load package in https://gowalker.org: %s' % r.text)

    if not d['cmd']:
        raise Exception('Repo: [%s] is a go-lib, this platform only support main package' % name)
    return d['synopsis']

@bp.route('/')
@models.db_session
def home():
    address = flask.request.args.get('address')
    if address:
        reponame = cleanname(address)
        if not models.Repo.get(name=reponame):
            try:
                desc = checkrepo(reponame)
{'fraction_non_alphanumeric': 0.16432865731462926, 'fraction_numerical': 0.0012024048096192384, 'mean_word_length': 2.7933130699088147, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7179125', 'n_tokens_mistral': 821, 'n_tokens_neox': 806, 'n_words': 218}