<commit_before>﻿/*
 * Copyright (c) 2013, SteamDB. All rights reserved.
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using MySql.Data.MySqlClient;
using SteamKit2;
using System.IO;
using Jayrock.Json;

namespace PICSUpdater
{
    class AppProcessor
    {
        public void ProcessApp(object appp)
        {
            Dictionary<string, string> appdata = new Dictionary<string, string>();
            KeyValuePair<UInt32, SteamApps.PICSProductInfoCallback.PICSProductInfo> app = (KeyValuePair<UInt32, SteamApps.PICSProductInfoCallback.PICSProductInfo>)appp;
            MySqlDataReader Reader = DbWorker.ExecuteReader(@"SELECT * FROM AppsInfo INNER JOIN KeyNames ON AppsInfo.Key=KeyNames.ID WHERE AppID = @AppId", new MySqlParameter[]
                {
                    new MySqlParameter("AppID", app.Value.ID)
                });
            while (Reader.Read())
            {
                appdata.Add(GetDBString("Name", Reader), GetDBString("Value", Reader));
            }
            Reader.Close();
            Reader.Dispose();

            String AppName = "";
            String AppType = "";

            MySqlDataReader MainReader = DbWorker.ExecuteReader(@"SELECT `Name`, `AppType` FROM Apps WHERE AppID = @AppID", new MySqlParameter[]
                {
                    new MySqlParameter("AppID", app.Value.ID)
                });

            if (MainReader.Read())
            {
                AppName = GetDBString("Name", MainReader);
                AppType = GetDBString("AppType", MainReader);
            }
            MainReader.Close();
            MainReader.Dispose();

            if (app.Value.KeyValues["common"]["name"].Value != null)
            {
                string newAppType = "0";
                if (!app.Value.KeyValues["common"]["type"].Equals(""))
                {
                    newAppType = getType(app.Value.KeyValues["common"]["type"].Value); // Value.ToString() ?? other part of code had this // also this is the only time getType() is used, you can get rid of it, if you want
                }

                if (AppName.Equals("") || AppName.StartsWith("SteamDB Unknown App"))
                {
                    DbWorker.ExecuteNonQuery("INSERT INTO Apps (AppID, AppType, Name) VALUES (@AppId, @Type, @AppName) ON DUPLICATE KEY UPDATE `Name` = @AppName, `AppType` = @Type", new MySqlParameter[] { new MySqlParameter("@AppId", app.Value.ID), new MySqlParameter("@Type", newAppType), new MySqlParameter("@AppName", app.Value.KeyValues["common"]["name"].Value.ToString()) });
                    MakeHistory(app.Value.ID, app.Value.ChangeNumber, "created_app");
                    MakeHistory(app.Value.ID, app.Value.ChangeNumber, "created_info", "10", "", app.Value.KeyValues["common"]["name"].Value.ToString(), true);
                }
                else if (!AppName.Equals(app.Value.KeyValues["common"]["name"].Value.ToString()))
                {
                    DbWorker.ExecuteNonQuery("UPDATE Apps SET Name = @AppName WHERE AppID = @AppId", new MySqlParameter[] { new MySqlParameter("@AppId", app.Value.ID), new MySqlParameter("@AppName", app.Value.KeyValues["common"]["name"].Value.ToString()) });
                    MakeHistory(app.Value.ID, app.Value.ChangeNumber, "modified_info", "10", AppName, app.Value.KeyValues["common"]["name"].Value.ToString(), true);
                }

                if (AppType.Equals("") || AppType.Equals("0"))
                {
                    MakeHistory(app.Value.ID, app.Value.ChangeNumber, "created_info", "9", "", newAppType, true);
                    DbWorker.ExecuteNonQuery("UPDATE Apps SET AppType = @type WHERE AppID = @AppId", new MySqlParameter[] { new MySqlParameter("@AppId", app.Value.ID), new MySqlParameter("@type", newAppType) });
                }

                else if (!AppType.Equals(newAppType))
                {
                    MakeHistory(app.Value.ID, app.Value.ChangeNumber, "modified_info", "9", AppType, newAppType, true);
                    DbWorker.ExecuteNonQuery("UPDATE Apps SET AppType = @type WHERE AppID = @AppId", new MySqlParameter[] { new MySqlParameter("@AppId", app.Value.ID), new MySqlParameter("@type", newAppType) });
                }
            }

            #region HugeQuery
            if (app.Value.KeyValues == null)
            {
                return;
            }

            foreach (KeyValue section in app.Value.KeyValues.Children)
            {
                String sectionName = section.Name;
                if (sectionName == "appid" || sectionName == "public_only")
                {
                    continue;
                }

                if (sectionName == "change_number")
                {
                    if (!appdata.ContainsKey("marlamin_change_number"))
                    {
                        DbWorker.ExecuteNonQuery("INSERT IGNORE INTO KeyNames(Name, DisplayName) VALUES('marlamin_change_number', 'change_number')");
                        MakeAppsInfo(app.Value.ID, "marlamin_change_number", section.Value);
                        MakeHistory(app.Value.ID, app.Value.ChangeNumber, "created_key", "marlamin_change_number", "", section.Value);
                    }
                    else
                    {
                        if (!appdata["marlamin_change_number"].Equals(section.Value))
                        {
                            MakeAppsInfo(app.Value.ID, "marlamin_change_number", section.Value);
                            MakeHistory(app.Value.ID, app.Value.ChangeNumber, "modified_key", "marlamin_change_number", appdata["marlamin_change_number"], section.Value);
                        }
                    }
                    appdata.Remove("marlamin_change_number");
                }else if (sectionName == "common" || sectionName == "extended"){
                    foreach (KeyValue keyvalue in section.Children)
                    {
                        String keynamecheck = sectionName.ToLower() + "_" + keyvalue.Name.ToString();
                        if (keynamecheck.Equals("common_type") || keynamecheck.Equals("common_gameid") || keynamecheck.Equals("common_name") || keynamecheck.Equals("extended_order"))
                        {
                            continue;
                        }
                        else if 
                        (
                        keynamecheck.Equals("extended_us ca mx pr bs pa bz cr sv gt hn ni ve co gy sr gf br pe bo py uy ar cl ec") ||
                        keynamecheck.Equals("extended_de") ||
                        keynamecheck.Equals("extended_jp") ||
                        keynamecheck.Equals("extended_af ax al dz as ad ao ai aq ag am aw at az au nz bh bd bb by be bj bm bt ba bw bv io bn bg bf bi kh cm cv ky cf td cn cx cc km cg cd ck ci hr cy cz dk dj dm do eg gq er ee et fk fo fj fi fr pf tf ga gm ge de gh gi gr gl gd gp gu gn gw gg ht hm va hk hu is in id ie im il it jm jp je jo kz ke ki kr kw kg la lv lb ls lr li lt lu mo mk mg mw my mv ml mt mh mq mr mu yt fm md mc mn me ms ma mz na nr np nl an nc nz ne ng nu nf mp no om pk pw ps pg ph pn pl pt qa re ro ru rw sh kn lc pm vc ws sm st sa sn rs sc sl sg sk si sb so za gs es lk sj sz se ch tw tj tz th tl tg tk to tt tn tr tm tc tv ug ua ae gb um uz vu vn vg vi wf eh ye zm zw") ||
                        keynamecheck.Equals("extended_us ca mx pr bs pa bz cr sv gt hn ni ve co gy sr gf br pe bo py uy ar cl ec") ||
                        keynamecheck.Equals("extended_im je gs gb uk fr de es ru it se fi no dk pl ch at ie nl be lu gg ru") ||
                        keynamecheck.Equals("extended_us ca mx pr") ||
                        keynamecheck.Equals("extended_cn") ||
                        keynamecheck.Equals("extended_im je gs gb gg uk") ||
                        keynamecheck.Equals("extended_us")
                        )
                        {
                            Console.WriteLine("<NAME>, why these long keynames? " + app.Value.ID + " " + keynamecheck);
                            continue;
                        }

                        List<KeyValue> subvalues = keyvalue.Children;
                        if (subvalues.Count == 0)
                        {
                            if (!keyvalue.Value.ToString().Equals(""))
                            {<commit_msg>Shaved off a few lines<commit_after>﻿/*
 * Copyright (c) 2013, SteamDB. All rights reserved.
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using MySql.Data.MySqlClient;
using SteamKit2;
using System.IO;
using Jayrock.Json;

namespace PICSUpdater
{
    class AppProcessor
    {
        public void ProcessApp(object appp)
        {
            Dictionary<string, string> appdata = new Dictionary<string, string>();
            KeyValuePair<UInt32, SteamApps.PICSProductInfoCallback.PICSProductInfo> app = (KeyValuePair<UInt32, SteamApps.PICSProductInfoCallback.PICSProductInfo>)appp;
            MySqlDataReader Reader = DbWorker.ExecuteReader(@"SELECT * FROM AppsInfo INNER JOIN KeyNames ON AppsInfo.Key=KeyNames.ID WHERE AppID = @AppId", new MySqlParameter[]
                {
                    new MySqlParameter("AppID", app.Value.ID)
                });
            while (Reader.Read())
            {
                appdata.Add(GetDBString("Name", Reader), GetDBString("Value", Reader));
            }
            Reader.Close();
            Reader.Dispose();

            String AppName = "";
            String AppType = "";

            MySqlDataReader MainReader = DbWorker.ExecuteReader(@"SELECT `Name`, `AppType` FROM Apps WHERE AppID = @AppID", new MySqlParameter[]
                {
                    new MySqlParameter("AppID", app.Value.ID)
                });

            if (MainReader.Read())
            {
                AppName = GetDBString("Name", MainReader);
                AppType = GetDBString("AppType", MainReader);
            }
            MainReader.Close();
            MainReader.Dispose();

            if (app.Value.KeyValues["common"]["name"].Value != null)
            {
                string newAppType = "0";
                if (!app.Value.KeyValues["common"]["type"].Equals(""))
                {
                    newAppType = getType(app.Value.KeyValues["common"]["type"].Value); // Value.ToString() ?? other part of code had this // also this is the only time getType() is used, you can get rid of it, if you want
                }

                if (AppName.Equals("") || AppName.StartsWith("SteamDB Unknown App"))
                {
                    DbWorker.ExecuteNonQuery("INSERT INTO Apps (AppID, AppType, Name) VALUES (@AppId, @Type, @AppName) ON DUPLICATE KEY UPDATE `Name` = @AppName, `AppType` = @Type", new MySqlParameter[] { new MySqlParameter("@AppId", app.Value.ID), new MySqlParameter("@Type", newAppType), new MySqlParameter("@AppName", app.Value.KeyValues["common"]["name"].Value.ToString()) });
                    MakeHistory(app.Value.ID, app.Value.ChangeNumber, "created_app");
                    MakeHistory(app.Value.ID, app.Value.ChangeNumber, "created_info", "10", "", app.Value.KeyValues["common"]["name"].Value.ToString(), true);
                }
                else if (!AppName.Equals(app.Value.KeyValues["common"]["name"].Value.ToString()))
                {
                    DbWorker.ExecuteNonQuery("UPDATE Apps SET Name = @AppName WHERE AppID = @AppId", new MySqlParameter[] { new MySqlParameter("@AppId", app.Value.ID), new MySqlParameter("@AppName", app.Value.KeyValues["common"]["name"].Value.ToString()) });
                    MakeHistory(app.Value.ID, app.Value.ChangeNumber, "modified_info", "10", AppName, app.Value.KeyValues["common"]["name"].Value.ToString(), true);
                }

                if (AppType.Equals("") || AppType.Equals("0"))
                {
                    MakeHistory(app.Value.ID, app.Value.ChangeNumber, "created_info", "9", "", newAppType, true);
                    DbWorker.ExecuteNonQuery("UPDATE Apps SET AppType = @type WHERE AppID = @AppId", new MySqlParameter[] { new MySqlParameter("@AppId", app.Value.ID), new MySqlParameter("@type", newAppType) });
                }

                else if (!AppType.Equals(newAppType))
                {
                    MakeHistory(app.Value.ID, app.Value.ChangeNumber, "modified_info", "9", AppType, newAppType, true);
                    DbWorker.ExecuteNonQuery("UPDATE Apps SET AppType = @type WHERE AppID = @AppId", new MySqlParameter[] { new MySqlParameter("@AppId", app.Value.ID), new MySqlParameter("@type", newAppType) });
                }
            }

            #region HugeQuery
            if (app.Value.KeyValues == null)
            {
                return;
            }

            foreach (KeyValue section in app.Value.KeyValues.Children)
            {
                String sectionName = section.Name;
                if (sectionName == "appid" || sectionName == "public_only")
                {
                    continue;
                }

                if (sectionName == "change_number")
                {
                    if (!appdata.ContainsKey("marlamin_change_number"))
                    {
                        DbWorker.ExecuteNonQuery("INSERT IGNORE INTO KeyNames(Name, DisplayName) VALUES('marlamin_change_number', 'change_number')");
                        MakeAppsInfo(app.Value.ID, "marlamin_change_number", section.Value);
                        MakeHistory(app.Value.ID, app.Value.ChangeNumber, "created_key", "marlamin_change_number", "", section.Value);
                    }
                    else
                    {
                        if (!appdata["marlamin_change_number"].Equals(section.Value))
                        {
                            MakeAppsInfo(app.Value.ID, "marlamin_change_number", section.Value);
                            MakeHistory(app.Value.ID, app.Value.ChangeNumber, "modified_key", "marlamin_change_number", appdata["marlamin_change_number"], section.Value);
                        }
                    }
                    appdata.Remove("marlamin_change_number");
                }else if (sectionName == "common" || sectionName == "extended"){
                    foreach (KeyValue keyvalue in section.Children)
                    {
                        String keynamecheck = sectionName.ToLower() + "_" + keyvalue.Name.ToString();
                        if (keynamecheck.Equals("common_type") || keynamecheck.Equals("common_gameid") || keynamecheck.Equals("common_name") || keynamecheck.Equals("extended_order"))
                        {
                            continue;
                        }
                        else if 
                        (
                        keynamecheck.StartsWith("extended_us ") ||
                        keynamecheck.StartsWith("extended_im ") ||
                        keynamecheck.Equals("extended_de") ||
                        keynamecheck.Equals("extended_jp") ||
                        keynamecheck.Equals("extended_af ax al dz as ad ao ai aq ag am aw at az au nz bh bd bb by be bj bm bt ba bw bv io bn bg bf bi kh cm cv ky cf td cn cx cc km cg cd ck ci hr cy cz dk dj dm do eg gq er ee et fk fo fj fi fr pf tf ga gm ge de gh gi gr gl gd gp gu gn gw gg ht hm va hk hu is in id ie im il it jm jp je jo kz ke ki kr kw kg la lv lb ls lr li lt lu mo mk mg mw my mv ml mt mh mq mr mu yt fm md mc mn me ms ma mz na nr np nl an nc nz ne ng nu nf mp no om pk pw ps pg ph pn pl pt qa re ro ru rw sh kn lc pm vc ws sm st sa sn rs sc sl sg sk si sb so za gs es lk sj sz se ch tw tj tz th tl tg tk to tt tn tr tm tc tv ug ua ae gb um uz vu vn vg vi wf eh ye zm zw") ||
                        keynamecheck.Equals("extended_cn") ||
                        keynamecheck.Equals("extended_us")
                        )
                        {
                            Console.WriteLine("<NAME>, why these long keynames? " + app.Value.ID + " " + keynamecheck);
                            continue;
                        }

                        List<KeyValue> subvalues = keyvalue.Children;
                        if (subvalues.Count == 0)
                        {
                            if (!keyvalue.Value.ToString().Equals(""))
                            {
{'fraction_non_alphanumeric': 0.10991600700948698, 'fraction_numerical': 0.002175358027675388, 'mean_word_length': 1.7491694352159468, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3896871', 'n_tokens_mistral': 4670, 'n_tokens_neox': 4428, 'n_words': 1249}