<commit_before>#!/usr/bin/env python3

# Something that I ran to extract module lists from copy pasted forum posts:
# cat new-list | tr '[' '\n' | tr ',' '\n' | sed -e 's/^[ \t]*//' | sed 's/ x.$//' | sort | uniq | ./market-stuff.py --filter

from collections import namedtuple
from xml.dom.minidom import parseString
import collections
import email
import sqlite3
import sys
import urllib.request as urlreq
import esi_load

CHUNK_SIZE = 100
ITEM_LIST = 'items'
EVECENTRAL_HOURS = 48
MARKET_HUB = 'Jita'

ColumnProperties = namedtuple('ColumnProperties', ['display_name', 'is_numeric'])
column_properties = collections.OrderedDict([
        ('Group',
         ColumnProperties(is_numeric=False, display_name='Group')),
        ('Item',
         ColumnProperties(is_numeric=False, display_name='Item')),
        ('Volume',
         ColumnProperties(is_numeric=True,  display_name='Volume')),
        ('Price',
         ColumnProperties(is_numeric=True,  display_name='Price')),
        ('HubVolume',
         ColumnProperties(is_numeric=True,  display_name='%s Volume' % MARKET_HUB)),
        ('HubPrice',
         ColumnProperties(is_numeric=True,  display_name='%s Price' % MARKET_HUB)),
        ('HubRelative',
         ColumnProperties(is_numeric=True,  display_name='Relative to %s' % MARKET_HUB)),
])

Item = namedtuple('Item', ['id', 'name', 'group', 'category', 'market_group_id'])
Row = namedtuple('Row', column_properties.keys())
MarketGroup = namedtuple('MarketGroup', ['id', 'parent_id', 'name', 'good_name'])

id2item = {}
name2item = {}
market_groups = {}
market_group_useful_names = {}

esi = None
conn = None
try:
    conn = sqlite3.connect('eve-dump.db')
    conn.text_factory = lambda x: str(x, 'latin1') # ASDF
    cursor = conn.cursor()
    cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
    if len(cursor.fetchall()) == 0:
        raise Exception("database is empty")
except Exception as e:
    print("Could not open Eve database dump:", str(e), file=sys.stderr)
    sys.exit(1)

def get_system_id(name):
    c = conn.cursor()
    c.execute("SELECT itemID from invnames where itemName = ?", (name,))
    return c.fetchone()[0]

def load_items():
    c = conn.cursor()
    c.execute("select typeId, typeName, groupName, categoryName, marketGroupID from invtypes join invgroups on invtypes.groupID = invgroups.groupID join invcategories on invgroups.categoryID = invcategories.categoryID")

    for entry in c:
        item = Item(*entry)
#        print(item)
        name2item[item.name] = item
        id2item[item.id] = item


def load_marketgroups():
    c = conn.cursor()
    c.execute("select marketGroupID, parentGroupID, marketGroupName from invmarketgroups")

    for entry in c:
        market_group = MarketGroup(*entry, good_name=None)
        market_groups[market_group.id] = market_group

    for (id, mg) in market_groups.items():
        market_groups[id] = mg._replace(good_name = useful_market_group_name(id))

# Get the list of all parents of a market group
def get_parents(id):
    trace = []
    while id:
        mg = market_groups[id]
        trace.append(mg.name)
        id = mg.parent_id
    return trace

# This, using some ad-hoc rules, tries to produce a useful category
# name.  There is nothing particularly principled about this, but I
# like the results.
def useful_market_group_name(id):
    # Grab the list of all of the parent market groups
    parents = list(reversed(get_parents(id)))
    if len(parents) == 1:
        return parents[0]
    elif parents[0] == 'Ship Modifications':
        if len(parents) >= 3 and parents[1] == 'Rigs':
            # Rig groups are named like "Electronics Superiority Rigs".
            # We want "Rigs - Electronics Superiority"
            name_body = parents[2].rsplit(None, 1)[0]
            rig_name = 'Rigs - ' + name_body
            return rig_name
        else: # This is just Subsystems, I think
            return parents[1]
    elif parents[0] == 'Ship Equipment':
        # Everything under "Ship Equipment" is a module... except for deployables
        if parents[1] == 'Deployable Equipment':
            return parents[1]
        else:
            # "Electronics and Sensor Upgrades" is really long and should be shorter.
            name = parents[1] if parents[1] != "Electronics and Sensor Upgrades" else "Electronics Upgrades"
            return 'Modules - ' + name
    elif parents[0] == 'Ships':
        # Get the real name for t2 ship classes
        if len(parents) >= 4 and parents[2].startswith("Advanced"):
            return 'Ships - ' + parents[3]
        else:
            return 'Ships - ' + parents[1]
    else:
        return parents[0]


def download_data(ids, system_id):
    base_url = 'http://api.evemarketer.com/ec/marketstat?hours=%d&usesystem=%d&' % (EVECENTRAL_HOURS, system_id)
    suffix = "&".join("typeid=%d" % i for i in ids)
    url = base_url + suffix
#    print(url)
    s = urlreq.urlopen(url)<commit_msg>Spoof the user-agent cause evemarketer is being difficult<commit_after>#!/usr/bin/env python3

# Something that I ran to extract module lists from copy pasted forum posts:
# cat new-list | tr '[' '\n' | tr ',' '\n' | sed -e 's/^[ \t]*//' | sed 's/ x.$//' | sort | uniq | ./market-stuff.py --filter

from collections import namedtuple
from xml.dom.minidom import parseString
import collections
import email
import sqlite3
import sys
import urllib.request as urlreq
import esi_load

CHUNK_SIZE = 100
ITEM_LIST = 'items'
EVECENTRAL_HOURS = 48
MARKET_HUB = 'Jita'

ColumnProperties = namedtuple('ColumnProperties', ['display_name', 'is_numeric'])
column_properties = collections.OrderedDict([
        ('Group',
         ColumnProperties(is_numeric=False, display_name='Group')),
        ('Item',
         ColumnProperties(is_numeric=False, display_name='Item')),
        ('Volume',
         ColumnProperties(is_numeric=True,  display_name='Volume')),
        ('Price',
         ColumnProperties(is_numeric=True,  display_name='Price')),
        ('HubVolume',
         ColumnProperties(is_numeric=True,  display_name='%s Volume' % MARKET_HUB)),
        ('HubPrice',
         ColumnProperties(is_numeric=True,  display_name='%s Price' % MARKET_HUB)),
        ('HubRelative',
         ColumnProperties(is_numeric=True,  display_name='Relative to %s' % MARKET_HUB)),
])

Item = namedtuple('Item', ['id', 'name', 'group', 'category', 'market_group_id'])
Row = namedtuple('Row', column_properties.keys())
MarketGroup = namedtuple('MarketGroup', ['id', 'parent_id', 'name', 'good_name'])

id2item = {}
name2item = {}
market_groups = {}
market_group_useful_names = {}

esi = None
conn = None
try:
    conn = sqlite3.connect('eve-dump.db')
    conn.text_factory = lambda x: str(x, 'latin1') # ASDF
    cursor = conn.cursor()
    cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
    if len(cursor.fetchall()) == 0:
        raise Exception("database is empty")
except Exception as e:
    print("Could not open Eve database dump:", str(e), file=sys.stderr)
    sys.exit(1)

def get_system_id(name):
    c = conn.cursor()
    c.execute("SELECT itemID from invnames where itemName = ?", (name,))
    return c.fetchone()[0]

def load_items():
    c = conn.cursor()
    c.execute("select typeId, typeName, groupName, categoryName, marketGroupID from invtypes join invgroups on invtypes.groupID = invgroups.groupID join invcategories on invgroups.categoryID = invcategories.categoryID")

    for entry in c:
        item = Item(*entry)
#        print(item)
        name2item[item.name] = item
        id2item[item.id] = item


def load_marketgroups():
    c = conn.cursor()
    c.execute("select marketGroupID, parentGroupID, marketGroupName from invmarketgroups")

    for entry in c:
        market_group = MarketGroup(*entry, good_name=None)
        market_groups[market_group.id] = market_group

    for (id, mg) in market_groups.items():
        market_groups[id] = mg._replace(good_name = useful_market_group_name(id))

# Get the list of all parents of a market group
def get_parents(id):
    trace = []
    while id:
        mg = market_groups[id]
        trace.append(mg.name)
        id = mg.parent_id
    return trace

# This, using some ad-hoc rules, tries to produce a useful category
# name.  There is nothing particularly principled about this, but I
# like the results.
def useful_market_group_name(id):
    # Grab the list of all of the parent market groups
    parents = list(reversed(get_parents(id)))
    if len(parents) == 1:
        return parents[0]
    elif parents[0] == 'Ship Modifications':
        if len(parents) >= 3 and parents[1] == 'Rigs':
            # Rig groups are named like "Electronics Superiority Rigs".
            # We want "Rigs - Electronics Superiority"
            name_body = parents[2].rsplit(None, 1)[0]
            rig_name = 'Rigs - ' + name_body
            return rig_name
        else: # This is just Subsystems, I think
            return parents[1]
    elif parents[0] == 'Ship Equipment':
        # Everything under "Ship Equipment" is a module... except for deployables
        if parents[1] == 'Deployable Equipment':
            return parents[1]
        else:
            # "Electronics and Sensor Upgrades" is really long and should be shorter.
            name = parents[1] if parents[1] != "Electronics and Sensor Upgrades" else "Electronics Upgrades"
            return 'Modules - ' + name
    elif parents[0] == 'Ships':
        # Get the real name for t2 ship classes
        if len(parents) >= 4 and parents[2].startswith("Advanced"):
            return 'Ships - ' + parents[3]
        else:
            return 'Ships - ' + parents[1]
    else:
        return parents[0]


def download_data(ids, system_id):
    base_url = 'https://api.evemarketer.com/ec/marketstat?hours=%d&usesystem=%d&' % (EVECENTRAL_HOURS, system_id)
    suffix = "&".join("typeid=%d" % i for i in ids)
    url = base_url + suffix
#    print(url)
    # evemarketer was blocking the urllib User-Agent (??), so lie
    req = urlreq.Request(url, data=None, headers={'User-Agent': 'wtf'})
    s = urlreq.urlopen(req)
{'fraction_non_alphanumeric': 0.1245674740484429, 'fraction_numerical': 0.00731586752347998, 'mean_word_length': 3.1975103734439836, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1251782', 'n_tokens_mistral': 3188, 'n_tokens_neox': 3107, 'n_words': 873}