<commit_before>// Metadata decoding

import std::ebml;
import std::ivec;
import std::option;
import std::vec;
import std::str;
import std::io;
import std::map::hashmap;
import syntax::ast;
import front::attr;
import middle::ty;
import common::*;
import tydecode::parse_def_id;
import tydecode::parse_ty_data;
import driver::session;
import syntax::print::pprust;
import cstore;

export get_symbol;
export get_tag_variants;
export get_type;
export get_type_param_count;
export lookup_defs;
export get_crate_attributes;
export list_crate_metadata;
export get_exported_metadata;

fn lookup_hash(&ebml::doc d, fn(vec[u8]) -> bool  eq_fn, uint hash) ->
   vec[ebml::doc] {
    auto index = ebml::get_doc(d, tag_index);
    auto table = ebml::get_doc(index, tag_index_table);
    auto hash_pos = table.start + hash % 256u * 4u;
    auto pos = ebml::be_uint_from_bytes(d.data, hash_pos, 4u);
    auto bucket = ebml::doc_at(d.data, pos);
    // Awkward logic because we can't ret from foreach yet

    let vec[ebml::doc] result = [];
    auto belt = tag_index_buckets_bucket_elt;
    for each (ebml::doc elt in ebml::tagged_docs(bucket, belt)) {
        auto pos = ebml::be_uint_from_bytes(elt.data, elt.start, 4u);
        if (eq_fn(vec::slice[u8](elt.data, elt.start + 4u, elt.end))) {
            vec::push(result, ebml::doc_at(d.data, pos));
        }
    }
    ret result;
}

fn maybe_find_item(int item_id, &ebml::doc items) -> option::t[ebml::doc] {
    fn eq_item(vec[u8] bytes, int item_id) -> bool {
        ret ebml::be_uint_from_bytes(bytes, 0u, 4u) as int == item_id;
    }<commit_msg>Remove non-existant export from metadata::decoder<commit_after>// Metadata decoding

import std::ebml;
import std::ivec;
import std::option;
import std::vec;
import std::str;
import std::io;
import std::map::hashmap;
import syntax::ast;
import front::attr;
import middle::ty;
import common::*;
import tydecode::parse_def_id;
import tydecode::parse_ty_data;
import driver::session;
import syntax::print::pprust;
import cstore;

export get_symbol;
export get_tag_variants;
export get_type;
export get_type_param_count;
export lookup_defs;
export get_crate_attributes;
export list_crate_metadata;

fn lookup_hash(&ebml::doc d, fn(vec[u8]) -> bool  eq_fn, uint hash) ->
   vec[ebml::doc] {
    auto index = ebml::get_doc(d, tag_index);
    auto table = ebml::get_doc(index, tag_index_table);
    auto hash_pos = table.start + hash % 256u * 4u;
    auto pos = ebml::be_uint_from_bytes(d.data, hash_pos, 4u);
    auto bucket = ebml::doc_at(d.data, pos);
    // Awkward logic because we can't ret from foreach yet

    let vec[ebml::doc] result = [];
    auto belt = tag_index_buckets_bucket_elt;
    for each (ebml::doc elt in ebml::tagged_docs(bucket, belt)) {
        auto pos = ebml::be_uint_from_bytes(elt.data, elt.start, 4u);
        if (eq_fn(vec::slice[u8](elt.data, elt.start + 4u, elt.end))) {
            vec::push(result, ebml::doc_at(d.data, pos));
        }
    }
    ret result;
}

fn maybe_find_item(int item_id, &ebml::doc items) -> option::t[ebml::doc] {
    fn eq_item(vec[u8] bytes, int item_id) -> bool {
        ret ebml::be_uint_from_bytes(bytes, 0u, 4u) as int == item_id;
    }
{'fraction_non_alphanumeric': 0.13873195229127433, 'fraction_numerical': 0.007532956685499058, 'mean_word_length': 4.494827586206896, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6787864', 'n_tokens_mistral': 1192, 'n_tokens_neox': 1165, 'n_words': 280}