<commit_before>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

'''The <structure> element.
'''

import os

from grit.node import base
from grit.node import variant

from grit import constants
from grit import exception
from grit import util

import grit.gather.admin_template
import grit.gather.chrome_html
import grit.gather.igoogle_strings
import grit.gather.muppet_strings
import grit.gather.policy_json
import grit.gather.rc
import grit.gather.tr_html
import grit.gather.txt

import grit.format.rc
import grit.format.rc_header

# Type of the gatherer to use for each type attribute
_GATHERERS = {
  'accelerators' : grit.gather.rc.Accelerators,
  'admin_template' : grit.gather.admin_template.AdmGatherer,
  'chrome_html' : grit.gather.chrome_html.ChromeHtml,
  'dialog'  : grit.gather.rc.Dialog,
  'igoogle' : grit.gather.igoogle_strings.IgoogleStrings,
  'menu'    : grit.gather.rc.Menu,
  'muppet'  : grit.gather.muppet_strings.MuppetStrings,
  'rcdata'  : grit.gather.rc.RCData,
  'tr_html' : grit.gather.tr_html.TrHtml,
  'txt'     : grit.gather.txt.TxtFile,
  'version' : grit.gather.rc.Version,
  'policy_template_metafile' : grit.gather.policy_json.PolicyJson,
}


# Formatter instance to use for each type attribute
# when formatting .rc files.
_RC_FORMATTERS = {
  'accelerators' : grit.format.rc.RcSection(),
  'admin_template' : grit.format.rc.RcInclude('ADM'),
  'chrome_html' : grit.format.rc.RcInclude('HTML'),
  'dialog'  : grit.format.rc.RcSection(),
  'igoogle' : grit.format.rc.RcInclude('XML'),
  'menu'    : grit.format.rc.RcSection(),
  'muppet'  : grit.format.rc.RcInclude('XML'),
  'rcdata'  : grit.format.rc.RcSection(),
  'tr_html' : grit.format.rc.RcInclude('HTML'),
  'txt'     : grit.format.rc.RcInclude('TXT'),
  'version' : grit.format.rc.RcSection(),
  'policy_template_metafile': None,
}


# TODO(joi) Print a warning if the 'variant_of_revision' attribute indicates
# that a skeleton variant is older than the original file.


class StructureNode(base.Node):
  '''A <structure> element.'''

  def __init__(self):
    base.Node.__init__(self)
    self.gatherer = None
    self.skeletons = {}  # expressions to skeleton gatherers

  def _IsValidChild(self, child):
    return isinstance(child, variant.SkeletonNode)

  def MandatoryAttributes(self):
    return ['type', 'name', 'file']

  def DefaultAttributes(self):
    return { 'encoding' : 'cp1252',
             'exclude_from_rc' : 'false',
             'line_end' : 'unix',
             'output_encoding' : 'utf-8',
             'generateid': 'true',
             'expand_variables' : 'false',
             'output_filename' : '',
             'fold_whitespace': 'false',
             'run_command' : '',
             # TODO(joi) this is a hack - should output all generated files
             # as SCons dependencies; however, for now there is a bug I can't
             # find where GRIT doesn't build the matching fileset, therefore
             # this hack so that only the files you really need are marked as
             # dependencies.
             'sconsdep' : 'false',
             }

  def IsExcludedFromRc(self):
    return self.attrs['exclude_from_rc'] == 'true'

  def GetLineEnd(self):
    '''Returns the end-of-line character or characters for files output because
    of this node ('\r\n', '\n', or '\r' depending on the 'line_end' attribute).
    '''
    if self.attrs['line_end'] == 'unix':
      return '\n'
    elif self.attrs['line_end'] == 'windows':
      return '\r\n'
    elif self.attrs['line_end'] == 'mac':
      return '\r'
    else:
      raise exception.UnexpectedAttribute(
        "Attribute 'line_end' must be one of 'linux' (default), 'windows' or 'mac'")

  def GetCliques(self):
    if self.gatherer:
      return self.gatherer.GetCliques()
    else:
      return []

  def GetDataPackPair(self, lang, encoding):
    """Returns a (id, string|None) pair that represents the resource id and raw
    bytes of the data (or None if no resource is generated).  This is used to
    generate the data pack data file.
    """
    from grit.format import rc_header
    id_map = rc_header.Item.tids_
    id = id_map[self.GetTextualIds()[0]]
    data = ''
    if self.gatherer:
      data = self.gatherer.GetData(lang, encoding)
    return id, data

  def GetTextualIds(self):
    if self.gatherer and self.attrs['type'] not in [
        'chrome_html', 'tr_html', 'admin_template', 'txt']:
      return self.gatherer.GetTextualIds()
    else:
      return [self.attrs['name']]

  def ItemFormatter(self, t):
    if t == 'rc_header':
      return grit.format.rc_header.Item()
    elif (t in ['rc_all', 'rc_translateable', 'rc_nontranslateable'] and
          self.SatisfiesOutputCondition()):
      return _RC_FORMATTERS[self.attrs['type']]
    else:
      return super(type(self), self).ItemFormatter(t)

  def RunGatherers(self, recursive=False, debug=False):
    if self.gatherer:
      return  # idempotent

    gathertype = _GATHERERS[self.attrs['type']]

    if debug:
      print 'Running gatherer %s for file %s' % (str(gathertype), self.FilenameToOpen())

    self.gatherer = gathertype.FromFile(self.FilenameToOpen(),
                                        self.attrs['name'],
                                        self.attrs['encoding'])
    self.gatherer.SetGrdNode(self)
    self.gatherer.SetUberClique(self.UberClique())
    if hasattr(self.GetRoot(), 'defines'):
      self.gatherer.SetDefines(self.GetRoot().defines)
    self.gatherer.SetAttributes(self.attrs)
    self.gatherer.Parse()

    for child in self.children:
      assert isinstance(child, variant.SkeletonNode)
      skel = gathertype.FromFile(child.FilenameToOpen(),
                                 self.attrs['name'],
                                 child.GetEncodingToUse())<commit_msg>Allow generating resource maps from structure nodes.<commit_after>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

'''The <structure> element.
'''

import os

from grit.node import base
from grit.node import variant

from grit import constants
from grit import exception
from grit import util

import grit.gather.admin_template
import grit.gather.chrome_html
import grit.gather.igoogle_strings
import grit.gather.muppet_strings
import grit.gather.policy_json
import grit.gather.rc
import grit.gather.tr_html
import grit.gather.txt

import grit.format.rc
import grit.format.rc_header

# Type of the gatherer to use for each type attribute
_GATHERERS = {
  'accelerators' : grit.gather.rc.Accelerators,
  'admin_template' : grit.gather.admin_template.AdmGatherer,
  'chrome_html' : grit.gather.chrome_html.ChromeHtml,
  'dialog'  : grit.gather.rc.Dialog,
  'igoogle' : grit.gather.igoogle_strings.IgoogleStrings,
  'menu'    : grit.gather.rc.Menu,
  'muppet'  : grit.gather.muppet_strings.MuppetStrings,
  'rcdata'  : grit.gather.rc.RCData,
  'tr_html' : grit.gather.tr_html.TrHtml,
  'txt'     : grit.gather.txt.TxtFile,
  'version' : grit.gather.rc.Version,
  'policy_template_metafile' : grit.gather.policy_json.PolicyJson,
}


# Formatter instance to use for each type attribute
# when formatting .rc files.
_RC_FORMATTERS = {
  'accelerators' : grit.format.rc.RcSection(),
  'admin_template' : grit.format.rc.RcInclude('ADM'),
  'chrome_html' : grit.format.rc.RcInclude('HTML'),
  'dialog'  : grit.format.rc.RcSection(),
  'igoogle' : grit.format.rc.RcInclude('XML'),
  'menu'    : grit.format.rc.RcSection(),
  'muppet'  : grit.format.rc.RcInclude('XML'),
  'rcdata'  : grit.format.rc.RcSection(),
  'tr_html' : grit.format.rc.RcInclude('HTML'),
  'txt'     : grit.format.rc.RcInclude('TXT'),
  'version' : grit.format.rc.RcSection(),
  'policy_template_metafile': None,
}


# TODO(joi) Print a warning if the 'variant_of_revision' attribute indicates
# that a skeleton variant is older than the original file.


class StructureNode(base.Node):
  '''A <structure> element.'''

  def __init__(self):
    base.Node.__init__(self)
    self.gatherer = None
    self.skeletons = {}  # expressions to skeleton gatherers

  def _IsValidChild(self, child):
    return isinstance(child, variant.SkeletonNode)

  def MandatoryAttributes(self):
    return ['type', 'name', 'file']

  def DefaultAttributes(self):
    return { 'encoding' : 'cp1252',
             'exclude_from_rc' : 'false',
             'line_end' : 'unix',
             'output_encoding' : 'utf-8',
             'generateid': 'true',
             'expand_variables' : 'false',
             'output_filename' : '',
             'fold_whitespace': 'false',
             'run_command' : '',
             # TODO(joi) this is a hack - should output all generated files
             # as SCons dependencies; however, for now there is a bug I can't
             # find where GRIT doesn't build the matching fileset, therefore
             # this hack so that only the files you really need are marked as
             # dependencies.
             'sconsdep' : 'false',
             }

  def IsExcludedFromRc(self):
    return self.attrs['exclude_from_rc'] == 'true'

  def GetLineEnd(self):
    '''Returns the end-of-line character or characters for files output because
    of this node ('\r\n', '\n', or '\r' depending on the 'line_end' attribute).
    '''
    if self.attrs['line_end'] == 'unix':
      return '\n'
    elif self.attrs['line_end'] == 'windows':
      return '\r\n'
    elif self.attrs['line_end'] == 'mac':
      return '\r'
    else:
      raise exception.UnexpectedAttribute(
        "Attribute 'line_end' must be one of 'linux' (default), 'windows' or 'mac'")

  def GetCliques(self):
    if self.gatherer:
      return self.gatherer.GetCliques()
    else:
      return []

  def GetDataPackPair(self, lang, encoding):
    """Returns a (id, string|None) pair that represents the resource id and raw
    bytes of the data (or None if no resource is generated).  This is used to
    generate the data pack data file.
    """
    from grit.format import rc_header
    id_map = rc_header.Item.tids_
    id = id_map[self.GetTextualIds()[0]]
    data = ''
    if self.gatherer:
      data = self.gatherer.GetData(lang, encoding)
    return id, data

  def GetTextualIds(self):
    if self.gatherer and self.attrs['type'] not in [
        'chrome_html', 'tr_html', 'admin_template', 'txt']:
      return self.gatherer.GetTextualIds()
    else:
      return [self.attrs['name']]

  def ItemFormatter(self, t):
    if t == 'rc_header':
      return grit.format.rc_header.Item()
    elif (t in ['rc_all', 'rc_translateable', 'rc_nontranslateable'] and
          self.SatisfiesOutputCondition()):
      return _RC_FORMATTERS[self.attrs['type']]
    elif t == 'resource_map_source':
      from grit.format import resource_map
      return resource_map.SourceInclude()
    elif t == 'resource_file_map_source':
      from grit.format import resource_map
      return resource_map.SourceFileInclude()
    else:
      return super(type(self), self).ItemFormatter(t)

  def RunGatherers(self, recursive=False, debug=False):
    if self.gatherer:
      return  # idempotent

    gathertype = _GATHERERS[self.attrs['type']]

    if debug:
      print 'Running gatherer %s for file %s' % (str(gathertype), self.FilenameToOpen())

    self.gatherer = gathertype.FromFile(self.FilenameToOpen(),
                                        self.attrs['name'],
                                        self.attrs['encoding'])
    self.gatherer.SetGrdNode(self)
    self.gatherer.SetUberClique(self.UberClique())
    if hasattr(self.GetRoot(), 'defines'):
      self.gatherer.SetDefines(self.GetRoot().defines)
    self.gatherer.SetAttributes(self.attrs)
    self.gatherer.Parse()

    for child in self.children:
      assert isinstance(child, variant.SkeletonNode)
      skel = gathertype.FromFile(child.FilenameToOpen(),
                                 self.attrs['name'],
                                 child.GetEncodingToUse())
{'fraction_non_alphanumeric': 0.12304526748971194, 'fraction_numerical': 0.0016460905349794238, 'mean_word_length': 3.0288461538461537, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6575103', 'n_tokens_mistral': 3956, 'n_tokens_neox': 3971, 'n_words': 947}