<commit_before>{-|
Netflix prize dataset

From the README :

The movie rating files contain over 100 million ratings from 480 thousand randomly-chosen, anonymous Netflix customers over 17 thousand movie titles.  The data were collected between October, 1998 and December, 2005 and reflect the distribution of all ratings received during this period.  The ratings are on a scale from 1 to 5 (integral) stars. To protect customer privacy, each customer id has been replaced with a randomly-assigned id.  The date of each rating and the title and year of release for each movie id are also provided.

The competition ended on September, 2009, and the dataset was subsequently removed from the public domain by the company. 

For further information, see <http://netflixprize.com/>
-}

module Numeric.Datasets.Netflix where

import Numeric.Datasets
import Data.Csv
<commit_msg>added file-embed example<commit_after>{-# language OverloadedStrings, TemplateHaskell #-}
{-|
Netflix prize dataset

From the README :

The movie rating files contain over 100 million ratings from 480 thousand randomly-chosen, anonymous Netflix customers over 17 thousand movie titles.  The data were collected between October, 1998 and December, 2005 and reflect the distribution of all ratings received during this period.  The ratings are on a scale from 1 to 5 (integral) stars. To protect customer privacy, each customer id has been replaced with a randomly-assigned id.  The date of each rating and the title and year of release for each movie id are also provided.

The competition ended on September, 2009, and the dataset was subsequently removed from the public domain by the company. 

For further information, see <http://netflixprize.com/>
-}

module Numeric.Datasets.Netflix where

import Numeric.Datasets
import Data.Csv
import Data.FileEmbed
import Data.ByteString

asdf :: [(FilePath, ByteString)]
asdf = $(embedDir "datafiles/")

{'fraction_non_alphanumeric': 0.051041666666666666, 'fraction_numerical': 0.022916666666666665, 'mean_word_length': 4.700296735905044, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7307156', 'n_tokens_mistral': 511, 'n_tokens_neox': 461, 'n_words': 265}