<commit_before># -*- coding:windows-1251 -*-

"""Find files with russian letters in specified directory

If files contain Russian letters they are considered untranslated
unless <!-- NLC --> marker is present at the same line in HTML code

"""
# pythonized by techtonik // gmail.com


import codecs
import os
import re
import sys

#sys.stdout.write("".decode("windows-1251"))

if len(sys.argv) < 2:
  print __doc__
  print "Usage: program <enc_eng_dir>"
  sys.exit()

# set encoding for console output
#print sys.stdout.encoding
#sys.stdout = codecs.lookup("windows-1251")[-1](sys.stdout)
#print sys.stdout.encoding


#reload(sys)
#sys.setdefaultencoding("cp866")

#print "".decode("cp1251")

#sw = codecs.lookup("cp866")[-1](sys.stdout)
#sw.write("sdf")


ru = re.compile("[--] *")
skip_mark = "<!-- NLC -->"
for root,dirs,files in os.walk(sys.argv[1]):
  if ".svn" in dirs:
    dirs.remove(".svn")
  for f in files:
    if f.endswith(".gif"):
      continue
    rucount = 0
    for l in open(os.path.join(root, f), "r"):
      if l.find(skip_mark) != -1:
        continue
      rutext = "".join(ru.findall(l))
      rucount += len(rutext)
      #if rutext: print rutext #.decode("cp1251")
    if rucount:
      print "%s - %d russian symbols" % (os.path.join(root, f), rucount)
      
  #print root,dirs,files<commit_msg>enc/tools/contrib/techtonik/untranslated.py - Update and ignore .git<commit_after>#!/usr/bin/env python2
# -*- coding:windows-1251 -*-

"""Find files with letters in russian windows-1251 encoding.

If English encyclopedia files contain Russian letters they are considered
untranslated unless <!-- NLC --> marker is present at the same line in
HTML code.

"""
# pythonized by techtonik // gmail.com


import codecs
import os
import re
import sys

#sys.stdout.write("".decode("windows-1251"))

if len(sys.argv) < 2:
  print __doc__
  print "Usage: program <dir>"
  sys.exit()

# set encoding for console output
#print sys.stdout.encoding
#sys.stdout = codecs.lookup("windows-1251")[-1](sys.stdout)
#print sys.stdout.encoding


#reload(sys)
#sys.setdefaultencoding("cp866")

#print "".decode("cp1251")

#sw = codecs.lookup("cp866")[-1](sys.stdout)
#sw.write("sdf")

dirs_exclude = ['.svn', '.git']

ru = re.compile("[--] *")
skip_mark = "<!-- NLC -->"
for root,dirs,files in os.walk(sys.argv[1]):

  # exclude dirs by modifying dirs in place
  dirs[:] = [d for d in dirs if not d in dirs_exclude]

  for f in files:
    if f.endswith(".gif"):
      continue
    rucount = 0
    for l in open(os.path.join(root, f), "r"):
      if l.find(skip_mark) != -1:
        continue
      rutext = "".join(ru.findall(l))
      rucount += len(rutext)
      #if rutext: print rutext #.decode("cp1251")
    if rucount:
      print "%s - %d russian symbols" % (os.path.join(root, f), rucount)
      
  #print root,dirs,files
{'fraction_non_alphanumeric': 0.16260162601626016, 'fraction_numerical': 0.024390243902439025, 'mean_word_length': 3.4149765990639627, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5990427', 'n_tokens_mistral': 1063, 'n_tokens_neox': 1013, 'n_words': 272}