<commit_before>#!/usr/bin/env python3

try:
    from systemd import journal
    systemd = True
except:
    systemd = False
from urllib.request import urlopen
from geojson import Feature, Point, FeatureCollection, dumps
try:
    from geojsonio import to_geojsonio
except:
    print("""==> Warning: You need to 'pip install github3.py' and download 'geojsonio.py'
    from https://github.com/jwass/geojsonio.py to this directory
    before you can use --geojsonio\n""")
    geojsonio = False
from simplekml import Kml


# Set the default config file location, this is overridden if the --config switch is used.
# If the --geojson or --kml switches are used, they will override the settings in the config file.
default_config = "/etc/archmap.conf"

# Set the output locations for users, geojson and kml.
# Setting 'default_geojson' or 'default_kml' to "no", will disable the output.
# These settings are overridden by the config file, if it file exsits.
default_users = "/tmp/users.txt"
default_geojson = "/tmp/output.geojson"
default_kml = "/tmp/output.kml"


# If the system uses the systemd journal, log to it. If the -v or --verbose
# flag is passed, print out info about what the script is doing.
def message(message):
    if systemd is not False:
        journal.send(message + ".", SYSLOG_IDENTIFIER="ArchMap")
    if args.verbose >= 1:
        print("==> " + message)


def get_users(output_file):
    """This funtion parses users from the ArchWiki and writes it to 'output_file'"""

    # Open and decode the ArchWiki page containing the list of users.
    message("Getting users from the ArchWiki")
    wiki = urlopen("https://wiki.archlinux.org/index.php/ArchMap/List")
    wiki_source = wiki.read().decode()

    # Grab the user data between the second set of <pre> tags.
    wiki_text_start = wiki_source.find('<pre>', wiki_source.find('<pre>') + 1) + 6
    wiki_text_end = wiki_source.find('</pre>', wiki_source.find('</pre>') + 1) - 1
    wiki_text = wiki_source[wiki_text_start:wiki_text_end]

    # Write the 'wiki_text' to 'output_file'.
    message("Writing users to " + output_file)
    wiki_output = open(output_file, 'w')
    wiki_output.write(wiki_text)
    wiki_output.close()


def parse_users(users_file):
    """This function parses the wiki text from 'users file' into it's components.
    It returns a list of lists containing the Latitude, Longitude, name and comment"""

    users = open(users_file, 'r')
    parsed = []

    for line in users:
        elements = line.split('"')

        coords = elements[0].strip(' ')
        coords = coords.split(',')
        latitude = float(coords[0])
        longitude = float(coords[1])
        name = elements[1].strip()
        comment = elements[2].strip()
        comment = comment[2:]

        parsed.append([latitude, longitude, name, comment])

    users.close()
    return parsed


def make_gis(parsed_users, output_file_geojson, output_file_kml, send_to_geojsonio):
    """This function reads the user data supplied by 'parsed_users', it then generates
    geojson and kml output and writes it to 'output_file_geojson' and 'output_file_kml'.

    If you set 'send_to_geojsonio' to 'True' it will send the raw geojson to geojson.io
    via a GitHub gist."""

    geojson = []
    kml = Kml()

    message("Making geosjon and kml")
<commit_msg>Add a message for parse_users() and fix comment<commit_after>#!/usr/bin/env python3

try:
    from systemd import journal
    systemd = True
except:
    systemd = False
from urllib.request import urlopen
from geojson import Feature, Point, FeatureCollection, dumps
try:
    from geojsonio import to_geojsonio
except:
    print("""==> Warning: You need to 'pip install github3.py' and download 'geojsonio.py'
    from https://github.com/jwass/geojsonio.py to this directory
    before you can use --geojsonio\n""")
    geojsonio = False
from simplekml import Kml


# Set the default config file location, this is overridden if the --config switch is used.
# If the --geojson or --kml switches are used, they will override the settings in the config file.
default_config = "/etc/archmap.conf"

# Set the output locations for users, geojson and kml.
# Setting 'default_geojson' or 'default_kml' to "no", will disable the output.
# These settings are overridden by the config file, if it file exsits.
default_users = "/tmp/users.txt"
default_geojson = "/tmp/output.geojson"
default_kml = "/tmp/output.kml"


def message(message):
    """This function takes a string in 'message'. If the system uses the systemd journal,
    log to it, using 'message'. If the -v or --verbose flag is passed, print out 'message'."""
    if systemd is not False:
        journal.send(message + ".", SYSLOG_IDENTIFIER="ArchMap")
    if args.verbose >= 1:
        print("==> " + message)


def get_users(output_file):
    """This funtion parses users from the ArchWiki and writes it to 'output_file'"""

    # Open and decode the ArchWiki page containing the list of users.
    message("Getting users from the ArchWiki")
    wiki = urlopen("https://wiki.archlinux.org/index.php/ArchMap/List")
    wiki_source = wiki.read().decode()

    # Grab the user data between the second set of <pre> tags.
    wiki_text_start = wiki_source.find('<pre>', wiki_source.find('<pre>') + 1) + 6
    wiki_text_end = wiki_source.find('</pre>', wiki_source.find('</pre>') + 1) - 1
    wiki_text = wiki_source[wiki_text_start:wiki_text_end]

    # Write the 'wiki_text' to 'output_file'.
    message("Writing users to " + output_file)
    wiki_output = open(output_file, 'w')
    wiki_output.write(wiki_text)
    wiki_output.close()


def parse_users(users_file):
    """This function parses the wiki text from 'users file' into it's components.
    It returns a list of lists containing the Latitude, Longitude, name and comment"""

    users = open(users_file, 'r')
    parsed = []

    message("Parsing ArchWiki text")
    for line in users:
        elements = line.split('"')

        coords = elements[0].strip(' ')
        coords = coords.split(',')
        latitude = float(coords[0])
        longitude = float(coords[1])
        name = elements[1].strip()
        comment = elements[2].strip()
        comment = comment[2:]

        parsed.append([latitude, longitude, name, comment])

    users.close()
    return parsed


def make_gis(parsed_users, output_file_geojson, output_file_kml, send_to_geojsonio):
    """This function reads the user data supplied by 'parsed_users', it then generates
    geojson and kml output and writes it to 'output_file_geojson' and 'output_file_kml'.

    If you set 'send_to_geojsonio' to 'True' it will send the raw geojson to geojson.io
    via a GitHub gist."""

    geojson = []
    kml = Kml()

    message("Making geosjon and kml")

{'fraction_non_alphanumeric': 0.10951605742193281, 'fraction_numerical': 0.0038478614769868287, 'mean_word_length': 3.484406104844061, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 19, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3452055', 'n_tokens_mistral': 2186, 'n_tokens_neox': 2079, 'n_words': 713}