<commit_before>#Imports
import pygeoip
import json
import re  
import argparse

#Set up argparse
parser = argparse.ArgumentParser(description="City-level geolocation")
parser.add_argument("-i", type = str, help = "The input JSON file", required = True, dest = "input")
parser.add_argument("-o", type = str, help = "The output JSON file", required = True, dest = "output")
args = parser.parse_args()

#Define city function
def city(x):
  
  #Read in files, caching in memory for speed
  ip4_geo = pygeoip.GeoIP(filename = "/usr/share/GeoIP/GeoIPCity.dat", flags = 1)
  ip6_geo = pygeoip.GeoIP(filename = "/usr/share/GeoIP/GeoLiteCityv6.dat", flags = 1)
  
  #Check type
  if not(isinstance(x,list)):
    x = [x]
  
  #Construct output list
  output = []
  
  #For each entry in the input list, retrieve the country code and add it to the output object
  for entry in x:
  
    if(bool(re.search(":",entry))):
      
      try:
        
        output.append(ip6_geo.record_by_addr(entry))
        
      except:
        
        output.append("Invalid")
        
    else:
      
      try:
        
        output.append(ip4_geo.record_by_addr(entry))
        
      except:
        
        output.append("Invalid")
        
  
  #Return
  return output
  
#Read in the file
json_data = open(args.input)
data = json.load(json_data, strict = False)
json_data.close()

#Run
output_data = city(x = data)

#Write out the results
file = open(args.output, 'w')
file.write(json.dumps(output_data))
file.close()<commit_msg>some geo_city changes (unstable and broken, but switching to new machine)<commit_after>#Imports
import pygeoip
import re  
import argparse

#Set up argparse
parser = argparse.ArgumentParser(description="City-level geolocation")
parser.add_argument("-i", type = str, help = "The input text file", required = True, dest = "input")
parser.add_argument("-o", type = str, help = "The output text file", required = True, dest = "output")
args = parser.parse_args()

#Read in MaxMind binary files, storing in memory for speed
ip4_geo = pygeoip.GeoIP(filename = "/usr/share/GeoIP/GeoIPCity.dat", flags = 1)
ip6_geo = pygeoip.GeoIP(filename = "/usr/share/GeoIP/GeoLiteCityv6.dat", flags = 1)

#Create object to iterate over
ip_list = []

#Open connection
file_con = open(name = args.input, mode = "r")

#Read in, handling invalid rows as we go
for line in file_con:
  try:
    ip_list.append(line)
  except:
    ip_list.append("")

#Close
file_con.close()

#Create output list
output = []

#For each entry, retrieve the country code and replace
for entry in ip_list:
  
  #If it's IPV6, use the 6 method
  if(bool(re.search(":",entry))):
    
    try:
      output.append(ip6_geo.record_by_addr(entry)+"\n")
    except:
      output.append("Invalid\n")
  
  #4, use 4.
  else:
    
    try:
      output.append(ip4_geo.record_by_addr(entry)+"\n")
    except:
      output.append("Invalid\n")

#Write out
output_file = open(name = args.output, mode = "w")
output_file.writelines(output)
output_file.close()
{'fraction_non_alphanumeric': 0.12450199203187251, 'fraction_numerical': 0.00597609561752988, 'mean_word_length': 3.3666666666666667, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6091893', 'n_tokens_mistral': 1038, 'n_tokens_neox': 1028, 'n_words': 278}