<commit_before>#!/usr/bin/env python

from lxml import html
import os
import sqlite3
import sys

DEBUG = os.getenv("DOCSET_DEBUG")

TO_DASH_TYPE = {
    "gd": "Guide",
    "fn": "Function",
    "trait": "Trait",
    "struct": "Struct",
    "structfield": "Field",
    "mod": "Module",
    "type": "Type",
    "static": "Constant",
    "macro": "Macro",
    "primitive": "Type",
    "ffi": "Function",
    "method": "Method",
    "field": "Field",
    "variant": "Variant",
    "enum": "Enum",
    "ffs": "Constant"
}


# Creates a full qualified name based on prefix and name
def make_fqn(prefix, name):
    return "%s::%s" % (prefix, name)


# Creates sqlite db
def prepare_index(db_path):
    if os.path.exists(db_path):
        os.remove(db_path)
    conn = sqlite3.connect(db_path)
    conn.execute("CREATE TABLE searchIndex(id INTEGER PRIMARY KEY, name TEXT, type TEXT, path TEXT);")
    cursor = conn.cursor()
    return (conn, cursor)


# Commits transaction and closes everything
def flush_index(idx):
    (conn, cursor) = idx
    cursor.close()
    conn.commit()
    conn.close()


# Adds an item to index
def add_to_index(idx, name, ty, path):
    if ty:
        dash_ty = TO_DASH_TYPE.get(ty, None)
        if not dash_ty:
            sys.stderr.write("Unknown type: %s, path %s" % (ty, path))
        else:
            (_, cursor) = idx
            cursor.execute("INSERT OR IGNORE INTO searchIndex(name, type, path) VALUES (?, ?, ?);", (name, dash_ty, path,))


def scrape(html_path, flt_list):
    try:
        tree = html.parse(html_path)

        def apply_filter(flt):
            return map(flt["func"], tree.xpath(flt["xpath"]))
<commit_msg>Skip empty docs silently<commit_after>#!/usr/bin/env python

from lxml import html
import os
import sqlite3
import sys

DEBUG = os.getenv("DOCSET_DEBUG")

TO_DASH_TYPE = {
    "gd": "Guide",
    "fn": "Function",
    "trait": "Trait",
    "struct": "Struct",
    "structfield": "Field",
    "mod": "Module",
    "type": "Type",
    "static": "Constant",
    "macro": "Macro",
    "primitive": "Type",
    "ffi": "Function",
    "method": "Method",
    "field": "Field",
    "variant": "Variant",
    "enum": "Enum",
    "ffs": "Constant"
}


# Creates a full qualified name based on prefix and name
def make_fqn(prefix, name):
    return "%s::%s" % (prefix, name)


# Creates sqlite db
def prepare_index(db_path):
    if os.path.exists(db_path):
        os.remove(db_path)
    conn = sqlite3.connect(db_path)
    conn.execute("CREATE TABLE searchIndex(id INTEGER PRIMARY KEY, name TEXT, type TEXT, path TEXT);")
    cursor = conn.cursor()
    return (conn, cursor)


# Commits transaction and closes everything
def flush_index(idx):
    (conn, cursor) = idx
    cursor.close()
    conn.commit()
    conn.close()


# Adds an item to index
def add_to_index(idx, name, ty, path):
    if ty:
        dash_ty = TO_DASH_TYPE.get(ty, None)
        if not dash_ty:
            sys.stderr.write("Unknown type: %s, path %s" % (ty, path))
        else:
            (_, cursor) = idx
            cursor.execute("INSERT OR IGNORE INTO searchIndex(name, type, path) VALUES (?, ?, ?);", (name, dash_ty, path,))


def scrape(html_path, flt_list):
    try:
        tree = html.parse(html_path)

        # Avoid exceptions on empty HTMLs
        if not tree.getroot():
            if DEBUG:
                print "Empty html at ", html_path
            return []
        def apply_filter(flt):
            return map(flt["func"], tree.xpath(flt["xpath"]))

{'fraction_non_alphanumeric': 0.15180102915951973, 'fraction_numerical': 0.0011435105774728416, 'mean_word_length': 2.845054945054945, 'pattern_counts': {'":': 32, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6902616', 'n_tokens_mistral': 1220, 'n_tokens_neox': 1175, 'n_words': 327}