<commit_before>#encoding: utf-8
from django.http import HttpResponse
from django.shortcuts import render_to_response
from django.core.paginator import Paginator
from django.template import RequestContext
try:
    import cmemcache as memcache
except:
    import memcache
import cjson
import hashlib
import os


def globalrequest(request):
    word,type_class = "",""
    menus = [
        "备孕",
        "怀孕",
        "产后",
        "0-1岁",
        "1-2岁",
        "2-3岁",
        "3-6岁",
        "专家咨询",
    ]
    return dict(word=word,
                type_class=type_class,
                menus=menus)
def hello(request):
    data = []
    mc = memcache.Client(['192.168.127.12:11212'])
    for m in menus:
        key = hashlib.md5(m).hexdigest()
        d = mc.get(key)
        if d:
            data.append( cjson.decode(d)[:4] )
    show_views=[
        "1918725321",
        "5710587117",
        "167035163130",
        "16549921536",
    ]
    data_view = []
    mc2 = memcache.Client(['192.168.127.12:11211'])
    for key in show_views:
        d = mc2.get(key)
        if d:
            d = cjson.decode(d)
            d["kid"]=key
            data_view.append( d )
            
    recoms=[
        "1918725321",
        "5710587117",
        "167035163130",
        "16549921536",
        "9389121638",
        "16549921536",
        "6065323270",
        "7181029938",
    ]
    data_recoms = []
    for key in recoms:
        d = mc2.get(key)
        if d:
            d = cjson.decode(d)
            d["kid"]=key
            data_recoms.append( d )
            
    return render_to_response('index.html',locals(),
            context_instance=RequestContext(request))
    
def link(request):
    return render_to_response('link.html',locals())

def keyword(request):
    word=request.GET["insearch"]
    page = 1
    if "page" in request.GET:
        page = request.GET["page"]
    type_class = "0"
    if "type_class" in request.GET:
        type_class=request.GET["type_class"]
    result=list(tmpsearch(word,type_class))
    p = Paginator(result,20)
    pp = p.page(page)
    result1=pp.object_list[:10]
    result2=pp.object_list[10:]
    return render_to_response('search.html',locals())

def v(request,kid):
    mc = memcache.Client(['192.168.127.12:11211'])
    obj=cjson.decode(mc.get(str(kid)))
    
    botlist = [
        'Baiduspider',
        'Googlebot',
        'Yahoo! Slurp',
    ]
    for bot in botlist:
        if request.META['HTTP_USER_AGENT'].find(bot)==0:
            obj['memo']=obj['addpinyin']
            break
        else:
            obj['memo']=obj['body']

    from ngram import ngram
    rel_page = []
    tg = ngram(menus,min_sim=0.0)
    words = "||".join( (tg.getSimilarStrings(obj["title"].encode("utf8"))).keys() )
    results=os.popen('dystmgr search -nl -max 10 /home/yanxu/khufu/khufu %s'%words).read()
    for kid2 in results.split('\n'):
        obj2=mc.get(kid2)
        if obj2==None:continue
        tmp=cjson.decode(obj2)
        rel_page.append( (tmp['title'],kid2) )
    return render_to_response('info.html',locals(),
        context_instance=RequestContext(request))

def tmpsearch(word,type_class):
    word=word.encode("utf8")
    type_class=type_class.encode("utf8")
    if type_class!="0":
        word = " ".join( (word,type_class) )
    mc = memcache.Client(['192.168.127.12:11211'])
    results=os.popen('dystmgr search -nl -max 200 /home/yanxu/khufu/khufu %s'%word).read()
    for kid in results.split('\n'):
        obj=mc.get(kid)
        if obj==None:continue
        tmp=cjson.decode(obj)
        yield tmp['title'],tmp['addpinyin'],kid
<commit_msg>将菜单的设置提取为公共变量<commit_after>#encoding: utf-8
from django.http import HttpResponse
from django.shortcuts import render_to_response
from django.core.paginator import Paginator
from django.template import RequestContext
try:
    import cmemcache as memcache
except:
    import memcache
import cjson
import hashlib
import os

menus = [
    "备孕",
    "怀孕",
    "产后",
    "0-1岁",
    "1-2岁",
    "2-3岁",
    "3-6岁",
    "专家咨询",
]

def globalrequest(request):
    word,type_class = "",""
    return dict(word=word,
                type_class=type_class,
                menus=menus)
def hello(request):
    data = []
    mc = memcache.Client(['192.168.127.12:11212'])
    for m in menus:
        key = hashlib.md5(m).hexdigest()
        d = mc.get(key)
        if d:
            data.append( cjson.decode(d)[:4] )
    show_views=[
        "1918725321",
        "5710587117",
        "167035163130",
        "16549921536",
    ]
    data_view = []
    mc2 = memcache.Client(['114.113.30.29:11211'])
    for key in show_views:
        d = mc2.get(key)
        if d:
            d = cjson.decode(d)
            d["kid"]=key
            data_view.append( d )
            
    recoms=[
        "1918725321",
        "5710587117",
        "167035163130",
        "16549921536",
        "9389121638",
        "16549921536",
        "6065323270",
        "7181029938",
    ]
    data_recoms = []
    for key in recoms:
        d = mc2.get(key)
        if d:
            d = cjson.decode(d)
            d["kid"]=key
            data_recoms.append( d )
            
    return render_to_response('index.html',locals(),
            context_instance=RequestContext(request))
    
def link(request):
    return render_to_response('link.html',locals())

def keyword(request):
    word=request.GET["insearch"]
    page = 1
    if "page" in request.GET:
        page = request.GET["page"]
    type_class = "0"
    if "type_class" in request.GET:
        type_class=request.GET["type_class"]
    result=list(tmpsearch(word,type_class))
    p = Paginator(result,20)
    pp = p.page(page)
    result1=pp.object_list[:10]
    result2=pp.object_list[10:]
    return render_to_response('search.html',locals())

def v(request,kid):
    mc = memcache.Client(['192.168.127.12:11211'])
    obj=cjson.decode(mc.get(str(kid)))
    
    botlist = [
        'Baiduspider',
        'Googlebot',
        'Yahoo! Slurp',
    ]
    for bot in botlist:
        if request.META['HTTP_USER_AGENT'].find(bot)==0:
            obj['memo']=obj['addpinyin']
            break
        else:
            obj['memo']=obj['body']

    from ngram import ngram
    rel_page = []
    tg = ngram(menus,min_sim=0.0)
    words = "||".join( (tg.getSimilarStrings(obj["title"].encode("utf8"))).keys() )
    results=os.popen('dystmgr search -nl -max 10 /home/yanxu/khufu/khufu %s'%words).read()
    for kid2 in results.split('\n'):
        obj2=mc.get(kid2)
        if obj2==None:continue
        tmp=cjson.decode(obj2)
        rel_page.append( (tmp['title'],kid2) )
    return render_to_response('info.html',locals(),
        context_instance=RequestContext(request))

def tmpsearch(word,type_class):
    word=word.encode("utf8")
    type_class=type_class.encode("utf8")
    if type_class!="0":
        word = " ".join( (word,type_class) )
    mc = memcache.Client(['192.168.127.12:11211'])
    results=os.popen('dystmgr search -nl -max 200 /home/yanxu/khufu/khufu %s'%word).read()
    for kid in results.split('\n'):
        obj=mc.get(kid)
        if obj==None:continue
        tmp=cjson.decode(obj)
        yield tmp['title'],tmp['addpinyin'],kid

{'fraction_non_alphanumeric': 0.14742967992240544, 'fraction_numerical': 0.06443120410142719, 'mean_word_length': 2.327800829875519, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 8, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3589978', 'n_tokens_mistral': 2898, 'n_tokens_neox': 2612, 'n_words': 442}