<commit_before>/**
 * @license
 * Copyright (c) 2015 The Polymer Project Authors. All rights reserved.
 * This code may only be used under the BSD style license found at
 * http://polymer.github.io/LICENSE.txt
 * The complete set of authors may be found at
 * http://polymer.github.io/AUTHORS.txt
 * The complete set of contributors may be found at
 * http://polymer.github.io/CONTRIBUTORS.txt
 * Code distributed by Google as part of the polymer project is also
 * subject to an additional IP rights grant found at
 * http://polymer.github.io/PATENTS.txt
 */

import * as fs from 'fs';
import * as jsonschema from 'jsonschema';
import * as pathLib from 'path';

import {Attribute, Element, ElementLike, ElementMixin, Elements, Namespace, Property, SourceRange} from './elements-format';
import {Namespace as ResolvedNamespace} from './javascript/namespace';
import {Document} from './model/document';
import {Attribute as ResolvedAttribute, Element as ResolvedElement, ElementMixin as ResolvedMixin, Property as ResolvedProperty, SourceRange as ResolvedSourceRange} from './model/model';
import {Package} from './model/package';

export type ElementOrMixin = ResolvedElement | ResolvedMixin;

export function generateElementMetadata(
    input: Package|Document[], packagePath: string): Elements {
  let elements: Set<ResolvedElement>;
  let mixins: Set<ResolvedMixin>;
  let namespaces: Set<ResolvedNamespace>;

  if (input instanceof Array) {
    elements = new Set();
    mixins = new Set();
    namespaces = new Set();
    for (const document of input as Document[]) {
      document.getByKind('element').forEach((f) => elements.add(f));
      document.getByKind('element-mixin').forEach((f) => mixins.add(f));
      document.getByKind('namespace').forEach((f) => namespaces.add(f));
    }
  } else {
    elements = input.getByKind('element');
    mixins = input.getByKind('element-mixin');
    namespaces = input.getByKind('namespace');
  }

  const metadata: Elements = {
    schema_version: '1.0.0',
  };

  if (namespaces.size > 0) {
    metadata.namespaces = Array.from(namespaces)
                              .map(
                                  (e) => serializeNamespace(
                                      e, packagePath)) as ResolvedNamespace[];
  }

  if (elements.size > 0) {
    metadata.elements = Array.from(elements).map(
        (e) => serializeElement(e, packagePath)) as Element[];
  }

  if (mixins.size > 0) {
    metadata.mixins = Array.from(mixins).map(
        (m) => serializeElementMixin(m, packagePath)) as ElementMixin[];
  }
<commit_msg>Add ability to filter features in metadata generation<commit_after>/**
 * @license
 * Copyright (c) 2015 The Polymer Project Authors. All rights reserved.
 * This code may only be used under the BSD style license found at
 * http://polymer.github.io/LICENSE.txt
 * The complete set of authors may be found at
 * http://polymer.github.io/AUTHORS.txt
 * The complete set of contributors may be found at
 * http://polymer.github.io/CONTRIBUTORS.txt
 * Code distributed by Google as part of the polymer project is also
 * subject to an additional IP rights grant found at
 * http://polymer.github.io/PATENTS.txt
 */

import * as fs from 'fs';
import * as jsonschema from 'jsonschema';
import * as pathLib from 'path';

import {Attribute, Element, ElementLike, ElementMixin, Elements, Namespace, Property, SourceRange} from './elements-format';
import {Namespace as ResolvedNamespace} from './javascript/namespace';
import {Document} from './model/document';
import {Feature} from './model/feature';
import {Attribute as ResolvedAttribute, Element as ResolvedElement, ElementMixin as ResolvedMixin, Property as ResolvedProperty, SourceRange as ResolvedSourceRange} from './model/model';
import {Package} from './model/package';

export type ElementOrMixin = ResolvedElement | ResolvedMixin;

export type Filter = (feature: Feature) => boolean;

export function generateElementMetadata(
    input: Package|Document[], packagePath: string, filter?: Filter): Elements {
  const _filter = filter || ((_: Feature) => true);

  let elements: Set<ResolvedElement>;
  let mixins: Set<ResolvedMixin>;
  let namespaces: Set<ResolvedNamespace>;

  if (input instanceof Array) {
    elements = new Set();
    mixins = new Set();
    namespaces = new Set();
    for (const document of input as Document[]) {
      Array.from(document.getByKind('element'))
          .filter(_filter)
          .forEach((f) => elements.add(f));
      Array.from(document.getByKind('element-mixin'))
          .filter(_filter)
          .forEach((f) => mixins.add(f));
      Array.from(document.getByKind('namespace'))
          .filter(_filter)
          .forEach((f) => namespaces.add(f));
    }
  } else {
    elements = new Set(Array.from(input.getByKind('element')).filter(_filter));
    mixins =
        new Set(Array.from(input.getByKind('element-mixin')).filter(_filter));
    namespaces =
        new Set(Array.from(input.getByKind('namespace')).filter(_filter));
  }

  const metadata: Elements = {
    schema_version: '1.0.0',
  };

  if (namespaces.size > 0) {
    metadata.namespaces = Array.from(namespaces)
                              .map(
                                  (e) => serializeNamespace(
                                      e, packagePath)) as ResolvedNamespace[];
  }

  if (elements.size > 0) {
    metadata.elements = Array.from(elements).map(
        (e) => serializeElement(e, packagePath)) as Element[];
  }

  if (mixins.size > 0) {
    metadata.mixins = Array.from(mixins).map(
        (m) => serializeElementMixin(m, packagePath)) as ElementMixin[];
  }

{'fraction_non_alphanumeric': 0.12542132339897108, 'fraction_numerical': 0.003547986517651233, 'mean_word_length': 3.373933281613654, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5291437', 'n_tokens_mistral': 1621, 'n_tokens_neox': 1581, 'n_words': 469}