<commit_before>﻿using DistrictsInTown.DbModel;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Data.Entity.Spatial;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;


namespace FourSquareImporter
{
    class Program
    {
        static void Main(string[] args)
        {
            ImportVenus("coffee", "Café");
        }

        private static void WriteVenues(IList<ForesquareVenue> venues)
        {
            foreach (var venue in venues)
                WriteVenue(venue);
        }

        private static void WriteVenue(ForesquareVenue venue)
        {
            Console.WriteLine("{0}  {1},{2} {3} {4}", venue.Name, venue.Longitude, venue.Latitude, venue.ZipCode, venue.Score);
        }

        private static void ImportVenus(string section, string keyword)
        {
            using (var dbContext = new DistrictsInTownModelContainer())
            {
                int chunkSize = 50;
                int offset = 0;

                ForesquareVenueResult result;
                do
                {
                    result = ExploreVenues(offset, chunkSize, "Berlin, DE", section, keyword).Result;
                    offset += chunkSize;

                    WriteVenues(result.Venues);

                    foreach (var venue in result.Venues)
                        AddVenueToDatabase(dbContext, venue);

                    try
                    {
                        dbContext.SaveChanges();
                    }
                    catch (Exception error)
                    {

                    }

                    Console.WriteLine("Total results: {0}", result.TotalResults);
                }
                while (offset < result.TotalResults);
            }
        }

        private static void AddVenueToDatabase(DistrictsInTownModelContainer dbContext, ForesquareVenue venue)
        {
            if (String.IsNullOrEmpty(venue.ZipCode))
                return;

            dbContext.Places.Add(new Places
            {
                Location = DbGeography.FromText(String.Format("POINT ({0} {1})", venue.Longitude, venue.Latitude)),
                Keyword = venue.Keyword,
                Score = venue.Score,
                Source = venue.Source,
                Zip = venue.ZipCode
            });
        }

        private static string CLIENT_ID = "AZQX4TS2BF0SKW5HEFVJANL5C2AMXFBKC3JPBMDZ1NBMLNX5";
        private static string CLIENT_SECRET = "<KEY>";

        class ForesquareVenue
        {
            public string Name;
            public double Longitude;
            public double Latitude;
            public double Score;
            public string Keyword;
            public string Source;
            public string ZipCode;
        }

        class ForesquareVenueResult
        {
            public long TotalResults;
            public IList<ForesquareVenue> Venues;
        }

        private static bool IsInCategory(dynamic categories, string categoryName)
        {
            foreach (dynamic category in categories)
            {
                if (category.shortName == categoryName)
                    return true;
            }

            return false;
        }

        private static async Task<ForesquareVenueResult> ExploreVenues(int offset, int limit, string near, string section, string category)
        {
            var httpClient = new HttpClient { BaseAddress = new Uri("https://api.foursquare.com/v2/") };

            var response = await httpClient.GetStringAsync("venues/explore?client_id=" + CLIENT_ID + "&client_secret=" + CLIENT_SECRET + "&offset=" + offset + "&limit=" + limit + "&near=" + near + "&section=" + section + "&v=20140101");

            dynamic responseJson = JsonConvert.DeserializeObject(response);

            var result = new ForesquareVenueResult();
            result.TotalResults = responseJson.response.totalResults;
            result.Venues = new List<ForesquareVenue>();

            foreach (var group in responseJson.response.groups)
            {
                if (group.name != "recommended")
                    continue;

                foreach (var groupItem in group.items)
                {
                    var venue = groupItem.venue;

                    try
                    {
                        /*if (!IsInCategory(venue.categories, category))
                            continue;*/

                        if (venue.rating < 7.0)
                            continue;

                        var venueResult = new ForesquareVenue
                        {
                            Name = venue.name,
                            Longitude = venue.location.lng,
                            Latitude = venue.location.lat,
                            ZipCode = venue.location.postalCode,
                            Keyword = "coffee",
                            Score = venue.rating,
                            Source = "foursquare_" + venue.id
                        };

                        result.Venues.Add(venueResult);
                    }
                    catch (Exception)
                    {
                        Console.WriteLine("Skipped {0}", venue.id);
                    }
                }
            }

            return result;
        }
    }
}<commit_msg>Importer can work with different keywords.<commit_after>﻿using DistrictsInTown.DbModel;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Data.Entity.Spatial;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;


namespace FourSquareImporter
{
    class Program
    {
        static void Main(string[] args)
        {
            ImportVenus("coffee", new[] { "Café", "Coffee Shops" }, "coffee");
            //ImportVenus("park", new[] { "Park", }, "park");
        }

        private static void WriteVenues(IList<ForesquareVenue> venues)
        {
            foreach (var venue in venues)
                WriteVenue(venue);
        }

        private static void WriteVenue(ForesquareVenue venue)
        {
            Console.WriteLine("{0}  {1},{2} {3} {4}", venue.Name, venue.Longitude, venue.Latitude, venue.ZipCode, venue.Score);
        }

        private static void ImportVenus(string section, string[] categories, string keyword)
        {
            using (var dbContext = new DistrictsInTownModelContainer())
            {
                int chunkSize = 50;
                int offset = 0;

                ForesquareVenueResult result;
                do
                {
                    result = ExploreVenues(offset, chunkSize, "Berlin, DE", section, categories, keyword).Result;
                    offset += chunkSize;

                    WriteVenues(result.Venues);

                    foreach (var venue in result.Venues)
                        AddVenueToDatabase(dbContext, venue);

                    try
                    {
                        dbContext.SaveChanges();
                    }
                    catch (Exception error)
                    {

                    }

                    Console.WriteLine("Total results: {0}", result.TotalResults);
                }
                while (offset < result.TotalResults);
            }
        }

        private static void AddVenueToDatabase(DistrictsInTownModelContainer dbContext, ForesquareVenue venue)
        {
            if (String.IsNullOrEmpty(venue.ZipCode))
                return;

            dbContext.Places.Add(new Places
            {
                Location = DbGeography.FromText(String.Format("POINT ({0} {1})", venue.Longitude, venue.Latitude)),
                Keyword = venue.Keyword,
                Score = venue.Score,
                Source = venue.Source,
                Zip = venue.ZipCode
            });
        }

        private static string CLIENT_ID = "AZQX4TS2BF0SKW5HEFVJANL5C2AMXFBKC3JPBMDZ1NBMLNX5";
        private static string CLIENT_SECRET = "<KEY>";

        class ForesquareVenue
        {
            public string Name;
            public double Longitude;
            public double Latitude;
            public double Score;
            public string Keyword;
            public string Source;
            public string ZipCode;
        }

        class ForesquareVenueResult
        {
            public long TotalResults;
            public IList<ForesquareVenue> Venues;
        }

        private static bool IsInCategory(dynamic categories, string[] categoryNames)
        {
            var names = new HashSet<string>(categoryNames);

            foreach (dynamic category in categories)
            {
                if (names.Contains((string)category.shortName))
                    return true;
            }

            Console.WriteLine("Skipped categories {0}", String.Join(" ", ((IEnumerable<dynamic>)categories).Select(n => n.shortName)));
            return false;
        }

        private static async Task<ForesquareVenueResult> ExploreVenues(int offset, int limit, string near, string section, string[] categories, string keyword)
        {
            var httpClient = new HttpClient { BaseAddress = new Uri("https://api.foursquare.com/v2/") };

            var response = await httpClient.GetStringAsync("venues/explore?client_id=" + CLIENT_ID + "&client_secret=" + CLIENT_SECRET + "&offset=" + offset + "&limit=" + limit + "&near=" + near + "&section=" + section + "&v=20140101");

            dynamic responseJson = JsonConvert.DeserializeObject(response);

            var result = new ForesquareVenueResult();
            result.TotalResults = responseJson.response.totalResults;
            result.Venues = new List<ForesquareVenue>();

            foreach (var group in responseJson.response.groups)
            {
                if (group.name != "recommended")
                    continue;

                foreach (var groupItem in group.items)
                {
                    var venue = groupItem.venue;

                    try
                    {
                        if (!IsInCategory(venue.categories, categories))
                            continue;

                        if (venue.rating < 7.0)
                            continue;

                        var venueResult = new ForesquareVenue
                        {
                            Name = venue.name,
                            Longitude = venue.location.lng,
                            Latitude = venue.location.lat,
                            ZipCode = venue.location.postalCode,
                            Keyword = keyword,
                            Score = venue.rating,
                            Source = "foursquare_" + venue.id
                        };

                        result.Venues.Add(venueResult);
                    }
                    catch (Exception)
                    {
                        Console.WriteLine("Skipped {0}", venue.id);
                    }
                }
            }

            return result;
        }
    }
}
{'fraction_non_alphanumeric': 0.08445366777948209, 'fraction_numerical': 0.0058648380402418114, 'mean_word_length': 1.3719238176760111, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 16, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '977421', 'n_tokens_mistral': 2862, 'n_tokens_neox': 2619, 'n_words': 609}