<commit_before>#!/usr/bin/env python2
# -*- coding:windows-1251 -*-

"""Find files with letters in russian windows-1251 encoding.

If English encyclopedia files contain Russian letters they are considered
untranslated unless <!-- NLC --> marker is present at the same line in
HTML code.

"""
# pythonized by techtonik // gmail.com


import codecs
import fnmatch
import os
import re
import sys

#sys.stdout.write("".decode("windows-1251"))

if len(sys.argv) < 2:
  print __doc__
  print "Usage: program <dir>"
  sys.exit()

# set encoding for console output
#print sys.stdout.encoding
#sys.stdout = codecs.lookup("windows-1251")[-1](sys.stdout)
#print sys.stdout.encoding


#reload(sys)
#sys.setdefaultencoding("cp866")

#print "".decode("cp1251")

#sw = codecs.lookup("cp866")[-1](sys.stdout)
#sw.write("sdf")

dirs_exclude = ['.svn', '.git']
files_exclude = ['*.gif', '*.png', '*.jpg', '*.exe', '*.ico', '*.msi', '*.rar']

# https://en.wikipedia.org/wiki/Windows-1251
cp1251 = re.compile(r"[\xA8\xB8\xC0-\xFF]+")
# by coincidence cp1251 codes match with lower
# byte of utf-8

skip_mark = "<!-- NLC -->"
for root,dirs,files in os.walk(sys.argv[1]):

  # exclude dirs by modifying dirs in place
  dirs[:] = [d for d in dirs if not d in dirs_exclude]

  for f in files:
    # exclude files by skipping them
    skip = False
    for pattern in files_exclude:
      if fnmatch.fnmatch(f.lower(), pattern):
        skip = True
    if skip:
      continue

    rucount = 0
    for l in open(os.path.join(root, f), "r"):
      if l.find(skip_mark) != -1:
        continue
      rutext = "".join(cp1251.findall(l))
      rucount += len(rutext)
      #if rutext: print rutext #.decode("cp1251")
    if rucount:
      print "%s - %d russian (cp1251) symbols" % (os.path.join(root, f), rucount)

  #print root,dirs,files<commit_msg>contrib: Update windows-1251 file scanner<commit_after>#!/usr/bin/env python2
# -*- coding:windows-1251 -*-

"""Find files with letters in russian windows-1251 encoding.

windows-1251 is a single byte encoding with a range 0xC0-0xFF
and 0xA8,0xB8 for symbols  and  respectfully. Unfortunately,
russian symbols in windows-1251 clash with russian symbols in
utf-8, where they take two bytes in the ranges:

 - 0xD081         
 - 0xD090-0xD0BF  -
 - 0xD180-0xD18F  -
 - 0xD191         

This code ignores lines with <!-- NLC --> marker.
"""
# pythonized by techtonik // gmail.com


import codecs
import fnmatch
import os
import re
import sys

#sys.stdout.write("".decode("windows-1251"))

if len(sys.argv) < 2:
  print(__doc__)
  print("Usage: program <dir>")
  sys.exit()

# set encoding for console output
#print sys.stdout.encoding
#sys.stdout = codecs.lookup("windows-1251")[-1](sys.stdout)
#print sys.stdout.encoding


#reload(sys)
#sys.setdefaultencoding("cp866")

#print "".decode("cp1251")

#sw = codecs.lookup("cp866")[-1](sys.stdout)
#sw.write("sdf")

dirs_exclude = ['.svn', '.git']
files_exclude = ['*.gif', '*.png', '*.jpg', '*.exe', '*.ico', '*.msi', '*.rar']

# https://en.wikipedia.org/wiki/Windows-1251
cp1251 = re.compile(r"[\xA8\xB8\xC0-\xFF]+")
# https://en.wikipedia.org/wiki/UTF-8#Encoding
utf8ru = re.compile(r"(\xD0[\x81\x90-\xBF]|\xD1[\x91\x80-\x8F])+")
utf8rest = re.compile(r"(\xC2[\xAB\xBB]|"  # angle quotes
                      r"\xE2\x80\x94|"     # long dash
                      r"\xC2\xA6)+")       # unicode |

skip_mark = "<!-- NLC -->"
for root,dirs,files in os.walk(sys.argv[1]):

  # exclude dirs by modifying dirs in place
  dirs[:] = [d for d in dirs if not d in dirs_exclude]

  for f in files:
    # exclude files by skipping them
    skip = False
    for pattern in files_exclude:
      if fnmatch.fnmatch(f.lower(), pattern):
        skip = True
    if skip:
      continue

    rucount = 0
    for i,l in enumerate(open(os.path.join(root, f), "r")):
      if l.find(skip_mark) != -1:
        continue
      noutf = utf8ru.sub('', l)  # remove russian utf-8 to avoid false positives
      noutf = utf8rest.sub('', noutf)  # remove other clashing utf-8 symbols
      rutext = "".join(cp1251.findall(noutf))
      rucount += len(rutext)
      if rutext:
          sys.stdout.write("line %3d: " % (i+1))
          print(noutf.encode('hex'))
          sys.stdout.write(rutext.decode("windows-1251").encode('utf-8'))
          #print(rutext.decode("cp1251"))
          sys.stdout.write("\n")

    if rucount:
      print("%s - %d russian (cp1251) symbols" % (os.path.join(root, f), rucount))

  #print root,dirs,files
{'fraction_non_alphanumeric': 0.16458240285841894, 'fraction_numerical': 0.042652970075926756, 'mean_word_length': 3.0978956999085088, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6525129', 'n_tokens_mistral': 1736, 'n_tokens_neox': 1611, 'n_words': 411}