<commit_before>#!/usr/bin/python
import sys, os, yaml, pickle, commands, time
from PyQt4 import QtCore, QtGui

def printYaml(data, level):
  indent_string = ''
  for i in xrange(0,level):
    indent_string += ' '
  print indent_string + str(data['name'])
  
  indent_string += ' '
  
  for item,value in data.items():
    if item != 'name' and item != 'parameters' and item != 'subblocks':
      print indent_string + item + ": " + str(value)
      
  if 'parameters' in data and data['parameters']:
    for param in data['parameters']:
      print indent_string + param['name']
      for name, value in param.items():
        print indent_string + ' ' + str(name) + ': ' + str(value)

  if data['subblocks']:
    for d in data['subblocks']:
      printYaml(d, level+1)    

class GenSyntax():
  def __init__(self, qt_app, app_path):
    self.qt_app = qt_app
    self.app_path = app_path
    self.saved_raw_data = None
    self.saved_data = None    
    
  def GetSyntax(self, recache):
    if not os.path.isfile(self.app_path):
      print 'ERROR: Executable ' + self.app_path + ' not found!'
      sys.exit(1)
    self.executable = os.path.basename(self.app_path)
    self.executable_path = os.path.dirname(self.app_path)
    yaml_dump_file_name = self.executable_path + '/yaml_dump_' + self.executable
    raw_yaml_dump_file_name = self.executable_path + '/yaml_dump_' + self.executable + '_raw'
    
    raw_data = self.getRawDump()

    if not self.saved_raw_data:
      if os.path.isfile(raw_yaml_dump_file_name):
        self.saved_raw_data = pickle.load(open(raw_yaml_dump_file_name, 'rb'))
      else:
        recache = True

    if not recache:
      if self.saved_raw_data != raw_data: # If the yaml has changed - force a recache
        recache = True
      elif self.saved_data: #If we have currently loaded data - just return it!
        return self.saved_data

    if recache or not os.path.exists(yaml_dump_file_name) or not os.path.exists(raw_yaml_dump_file_name):
      progress = QtGui.QProgressDialog("Recaching Syntax...", "Abort", 0, 10, None)
      progress.setWindowModality(QtCore.Qt.WindowModal)
      progress.show()
      progress.raise_()

      for i in xrange(0,7):
        progress.setValue(i)
        self.qt_app.processEvents()
        self.qt_app.flush()

      pickle.dump(raw_data, open(raw_yaml_dump_file_name, 'wb'))
      self.saved_raw_data = raw_data
      
      print "\nRecaching syntax. Subsequent startup times will be greatly reduced."
      
      data = yaml.load(raw_data)
      pickle.dump(data, open(yaml_dump_file_name, 'wb'))

      progress.setValue(8)
      progress.setValue(9)
      progress.setValue(10)
    else:<commit_msg>turn off printing for caching<commit_after>#!/usr/bin/python
import sys, os, yaml, pickle, commands, time
from PyQt4 import QtCore, QtGui

def printYaml(data, level):
  indent_string = ''
  for i in xrange(0,level):
    indent_string += ' '
  print indent_string + str(data['name'])
  
  indent_string += ' '
  
  for item,value in data.items():
    if item != 'name' and item != 'parameters' and item != 'subblocks':
      print indent_string + item + ": " + str(value)
      
  if 'parameters' in data and data['parameters']:
    for param in data['parameters']:
      print indent_string + param['name']
      for name, value in param.items():
        print indent_string + ' ' + str(name) + ': ' + str(value)

  if data['subblocks']:
    for d in data['subblocks']:
      printYaml(d, level+1)    

class GenSyntax():
  def __init__(self, qt_app, app_path):
    self.qt_app = qt_app
    self.app_path = app_path
    self.saved_raw_data = None
    self.saved_data = None    
    
  def GetSyntax(self, recache):
    if not os.path.isfile(self.app_path):
      print 'ERROR: Executable ' + self.app_path + ' not found!'
      sys.exit(1)
    self.executable = os.path.basename(self.app_path)
    self.executable_path = os.path.dirname(self.app_path)
    yaml_dump_file_name = self.executable_path + '/yaml_dump_' + self.executable
    raw_yaml_dump_file_name = self.executable_path + '/yaml_dump_' + self.executable + '_raw'
    
    raw_data = self.getRawDump()

    if not self.saved_raw_data:
      if os.path.isfile(raw_yaml_dump_file_name):
        self.saved_raw_data = pickle.load(open(raw_yaml_dump_file_name, 'rb'))
      else:
        recache = True

    if not recache:
      if self.saved_raw_data != raw_data: # If the yaml has changed - force a recache
        recache = True
      elif self.saved_data: #If we have currently loaded data - just return it!
        return self.saved_data

    if recache or not os.path.exists(yaml_dump_file_name) or not os.path.exists(raw_yaml_dump_file_name):
      progress = QtGui.QProgressDialog("Recaching Syntax...", "Abort", 0, 10, None)
      progress.setWindowModality(QtCore.Qt.WindowModal)
      progress.show()
      progress.raise_()

      for i in xrange(0,7):
        progress.setValue(i)
        self.qt_app.processEvents()
        self.qt_app.flush()

      pickle.dump(raw_data, open(raw_yaml_dump_file_name, 'wb'))
      self.saved_raw_data = raw_data
      
      data = yaml.load(raw_data)
      pickle.dump(data, open(yaml_dump_file_name, 'wb'))

      progress.setValue(8)
      progress.setValue(9)
      progress.setValue(10)
    else:
{'fraction_non_alphanumeric': 0.11690273843248347, 'fraction_numerical': 0.004910292728989613, 'mean_word_length': 3.01516300227445, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1964967', 'n_tokens_mistral': 1815, 'n_tokens_neox': 1785, 'n_words': 401}