<commit_before># -*- coding: utf-8 -*-

from bs4 import BeautifulSoup
import logging
import os
import re

# The "APH" airport code corresponds to the Jacksonville International
# Airport in Jacksonville, FL, United Sates.

# Limit the entries to the code (e.g. 'JAX' ) and the airport name +
# 'code' (e.g. 'Jacksonville International Airport code'). The latter
# being a redirect to the former. We could also include one with the
# word 'airport' removed (e.g. 'Jacksonville International code').
# Having the result for the city name would cover too many searches
# that aren't looking for the airport code.

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()

ENCODING='utf-8'
WIKIPEDIA_URL = 'https://wikipedia.org'
WIKIPEDIA_LIST_URL = 'https://en.wikipedia.org/wiki/List_of_airports_by_IATA_code:_'

def append_period(text):
	""" Append a period at the end of the sentence"""
	if text[-1] == '\"':
		return text[0:-1]+'.\"'
	return text

def is_international_airport(text):
	""" Returns the airport name without 'International Airport' or None """
	index = text.rfind(' International Airport')
	if index > 0:
		return text[0:index]
	else:
		return None

class Airport(object):
	""" Contains informations about an Airport"""
	def __init__(self, name, iata, icao, location, index_letter):
		self.name = name
		self.iata = iata
		self.icao = icao
		self.location = location
		self.index_letter = index_letter 

	def add_iata(self,fields,abstract,output):
		if self.iata != None and len(self.iata) != 0:
			fields[0] = self.iata
			fields[1] = 'A'
			fields[11] = append_period(abstract)
			output.append('%s' % ('\t'.join(fields)))

	def add_icao(self,fields,abstract,output):
		if self.icao != None and len(self.icao) != 0:
			fields[0] = self.icao
			fields[1] = 'A'
			fields[11] = append_period(abstract)
			output.append('%s' % ('\t'.join(fields)))

	def add_name(self,fields,abstract,output):
		if self.name != None and len(self.name) != "":
			fields[0] = self.name
			fields[1] = 'A'
			fields[11] = append_period(abstract)
			output.append('%s' % ('\t'.join(fields)))

	def add_location(self,fields,abstract,output):
		if self.location != None:
			location_names = self.location.split(',')
			if len(location_names) > 0:
				airport_location_name = location_names[0]+' Airport'
				if airport_location_name != self.name:
					fields[0] = airport_location_name
					fields[1] = 'A'
					fields[11] = append_period(abstract)
					output.append('%s' % ('\t'.join(fields)))

	def add_redirects(self,fields,output):
		name = is_international_airport(self.name_with_airport)
		if name != None:
			fields[11] = ''
			fields[12] = ''
			if self.iata != None:
				fields[0] = name
				fields[1] = 'R'
				fields[2] = self.name_with_airport
				output.append('%s' % ('\t'.join(fields)))
			if self.icao != None:
				fields[0] = name
				fields[1] = 'R'
				fields[2] = self.name_with_airport
				output.append('%s' % ('\t'.join(fields)))

	def __str__(self):
		output = []
		logger.debug(self.name+';'+self.iata+';'+self.icao+';'+self.location+';'+self.index_letter)
		fields = [
				'',	 # $unique_name<commit_msg>Fixed the redirections...<commit_after># -*- coding: utf-8 -*-

from bs4 import BeautifulSoup
import logging
import os
import re

# The "APH" airport code corresponds to the Jacksonville International
# Airport in Jacksonville, FL, United Sates.

# Limit the entries to the code (e.g. 'JAX' ) and the airport name +
# 'code' (e.g. 'Jacksonville International Airport code'). The latter
# being a redirect to the former. We could also include one with the
# word 'airport' removed (e.g. 'Jacksonville International code').
# Having the result for the city name would cover too many searches
# that aren't looking for the airport code.

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()

ENCODING='utf-8'
WIKIPEDIA_URL = 'https://wikipedia.org'
WIKIPEDIA_LIST_URL = 'https://en.wikipedia.org/wiki/List_of_airports_by_IATA_code:_'

def append_period(text):
	""" Append a period at the end of the sentence"""
	if text[-1] == '\"':
		return text[0:-1]+'.\"'
	return text

def is_international_airport(text):
	""" Returns the airport name without 'International Airport' or None """
	index = text.rfind(' International Airport')
	if index > 0:
		return text[0:index]
	else:
		return None

class Airport(object):
	""" Contains informations about an Airport"""
	def __init__(self, name, iata, icao, location, index_letter):
		self.name = name
		self.iata = iata
		self.icao = icao
		self.location = location
		self.index_letter = index_letter 

	def add_iata(self,fields,abstract,output):
		if self.iata != None and len(self.iata) != 0:
			fields[0] = self.iata
			fields[1] = 'A'
			fields[11] = append_period(abstract)
			output.append('%s' % ('\t'.join(fields)))

	def add_icao(self,fields,abstract,output):
		if self.icao != None and len(self.icao) != 0:
			fields[0] = self.icao
			fields[1] = 'A'
			fields[11] = append_period(abstract)
			output.append('%s' % ('\t'.join(fields)))

	def add_name(self,fields,abstract,output):
		if self.name != None and len(self.name) != "":
			fields[0] = self.name
			fields[1] = 'A'
			fields[11] = append_period(abstract)
			output.append('%s' % ('\t'.join(fields)))

	def add_location(self,fields,abstract,output):
		if self.location != None:
			location_names = self.location.split(',')
			if len(location_names) > 0:
				airport_location_name = location_names[0]+' Airport'
				if airport_location_name != self.name:
					fields[0] = airport_location_name
					fields[1] = 'A'
					fields[11] = append_period(abstract)
					output.append('%s' % ('\t'.join(fields)))

	def add_redirects(self,fields,output):
		name = is_international_airport(self.name_with_airport)
		if name != None:
			fields[0] = name
			fields[1] = 'R'
			fields[2] = self.name_with_airport
			fields[11] = ''
			fields[12] = ''
			output.append('%s' % ('\t'.join(fields)))

	def __str__(self):
		output = []
		logger.debug(self.name+';'+self.iata+';'+self.icao+';'+self.location+';'+self.index_letter)
		fields = [
				'',	 # $unique_name
{'fraction_non_alphanumeric': 0.15273680760406425, 'fraction_numerical': 0.011963290724352671, 'mean_word_length': 4.2073378839590445, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4404769', 'n_tokens_mistral': 2349, 'n_tokens_neox': 2099, 'n_words': 521}