<commit_before>import re
import socket
import cPickle as pickle
from logging import getLogger
from pprint import pformat
from itertools import islice, chain

import markdown
from pylons import tmpl_context as c, app_globals as g
from pysolr import SolrError

from . import helpers as h
from .markdown_extensions import ForgeExtension

log = getLogger(__name__)

def solarize(obj):
    if obj is None: return None
    doc = obj.index()
    if doc is None: return None
    # if index() returned doc without text, assume empty text
    doc['text'] = doc.pop('text', '')
    return doc

class SearchError(SolrError):
    pass

def search(q,short_timeout=False,ignore_errors=True,**kw):
    try:
        if short_timeout:
            return g.solr_short_timeout.search(q, **kw)
        else:
            return g.solr.search(q, **kw)
    except (SolrError, socket.error) as e:
        log.exception('Error in solr search')
        if not ignore_errors:
            match = re.search(r'<pre>(.*)</pre>', str(e))
            raise SearchError('Error running search query: %s' % (match.group(1) if match else e))

def search_artifact(atype, q, history=False, rows=10, short_timeout=False, **kw):
    """Performs SOLR search.

    Raises SearchError if SOLR returns an error.
    """
    # first, grab an artifact and get the fields that it indexes
    a = atype.query.find().first()
    if a is None: return # if there are no instance of atype, we won't find anything
    fields = a.index()
    # Now, we'll translate all the fld:
    q = atype.translate_query(q, fields)
    fq = [
        'type_s:%s' % fields['type_s'],
        'project_id_s:%s' % c.project._id,
        'mount_point_s:%s' % c.app.config.options.mount_point ]
    if not history:
        fq.append('is_history_b:False')
    return search(q, fq=fq, rows=rows, short_timeout=short_timeout, ignore_errors=False, **kw)

def find_shortlinks(text):
    md = markdown.Markdown(
        extensions=['codehilite', ForgeExtension(), 'tables'],
        output_format='html4')
    md.convert(text)
    link_index = md.treeprocessors['links'].alinks
    return [ link for link in link_index if link is not None]
<commit_msg>[#2835] ticket:288 Always use an empty string instead of None in `text` field<commit_after>import re
import socket
import cPickle as pickle
from logging import getLogger
from pprint import pformat
from itertools import islice, chain

import markdown
from pylons import tmpl_context as c, app_globals as g
from pysolr import SolrError

from . import helpers as h
from .markdown_extensions import ForgeExtension

log = getLogger(__name__)

def solarize(obj):
    if obj is None: return None
    doc = obj.index()
    if doc is None: return None
    # if index() returned doc without text, assume empty text
    if not doc.get('text'):
        doc['text'] = ''
    return doc

class SearchError(SolrError):
    pass

def search(q,short_timeout=False,ignore_errors=True,**kw):
    try:
        if short_timeout:
            return g.solr_short_timeout.search(q, **kw)
        else:
            return g.solr.search(q, **kw)
    except (SolrError, socket.error) as e:
        log.exception('Error in solr search')
        if not ignore_errors:
            match = re.search(r'<pre>(.*)</pre>', str(e))
            raise SearchError('Error running search query: %s' % (match.group(1) if match else e))

def search_artifact(atype, q, history=False, rows=10, short_timeout=False, **kw):
    """Performs SOLR search.

    Raises SearchError if SOLR returns an error.
    """
    # first, grab an artifact and get the fields that it indexes
    a = atype.query.find().first()
    if a is None: return # if there are no instance of atype, we won't find anything
    fields = a.index()
    # Now, we'll translate all the fld:
    q = atype.translate_query(q, fields)
    fq = [
        'type_s:%s' % fields['type_s'],
        'project_id_s:%s' % c.project._id,
        'mount_point_s:%s' % c.app.config.options.mount_point ]
    if not history:
        fq.append('is_history_b:False')
    return search(q, fq=fq, rows=rows, short_timeout=short_timeout, ignore_errors=False, **kw)

def find_shortlinks(text):
    md = markdown.Markdown(
        extensions=['codehilite', ForgeExtension(), 'tables'],
        output_format='html4')
    md.convert(text)
    link_index = md.treeprocessors['links'].alinks
    return [ link for link in link_index if link is not None]

{'fraction_non_alphanumeric': 0.10937853107344633, 'fraction_numerical': 0.003389830508474576, 'mean_word_length': 3.2680810028929606, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1523559', 'n_tokens_mistral': 1433, 'n_tokens_neox': 1401, 'n_words': 442}