<commit_before>pyspark -master local

productsRDD = sc.textFile("sqoop_import/products/")

# Products table desc from hive
#product_id              int
#product_category_id     int
#product_name            string
#product_description     string
#product_price           double
#product_image           string

prodByCatIDRDD = productsRDD.map(lambda x:(int(x.split(",")[1]),x)).groupByKey()
for i in prodByCatIDRDD.map(lambda x: list(x[1])).take(1)[0]: print i

# sort products by price(char)
for i in prodByCatIDRDD.flatMap(lambda x:sorted(x[1],key=lambda l:l.split(",")[4])).take(5): print i

#19,2,Nike Men's Fingertrap Max Training Shoe,,124.99,http://images.acmesports.sports/Nike+Men%27s+Fingertrap+Max+Training+Shoe
#2,2,Under Armour Men's Highlight MC Football Clea,,129.99,http://images.acmesports.sports/Under+Armour+Men%27s+Highlight+MC+Football+Cleat
#8,2,Nike Men's Vapor Carbon Elite TD Football Cle,,129.99,http://images.acmesports.sports/Nike+Men%27s+Vapor+Carbon+Elite+TD+Football+Cleat
#10,2,Under Armour Men's Highlight MC Football Clea,,129.99,http://images.acmesports.sports/Under+Armour+Men%27s+Highlight+MC+Football+Cleat
#17,2,Under Armour Men's Highlight MC Football Clea,,129.99,http://images.acmesports.sports/Under+Armour+Men%27s+Highlight+MC+Football+Cleat

# sort products by price(float, asc)
for i in prodByCatIDRDD.flatMap(lambda x:sorted(x[1],key=lambda l:float(l.split(",")[4]))).take(5): print i

#18,2,Reebok Men's Full Zip Training Jacket,,29.97,http://images.acmesports.sports/Reebok+Men%27s+Full+Zip+Training+Jacket
#22,2,Kijaro Dual Lock Chair,,29.99,http://images.acmesports.sports/Kijaro+Dual+Lock+Chair
#9,2,Nike Adult Vapor Jet 3.0 Receiver Gloves,,50.0,http://images.acmesports.sports/Nike+Adult+Vapor+Jet+3.0+Receiver+Gloves
#21,2,Under Armour Kids' Highlight RM Football Clea,,54.99,http://images.acmesports.sports/Under+Armour+Kids%27+Highlight+RM+Football+Cleat
#1,2,Quest Q64 10 FT. x 10 FT. Slant Leg Instant U,,59.98,http://images.acmesports.sports/Quest+Q64+10+FT.+x+10+FT.+Slant+Leg+Instant+Up+Canopy

# sort products by price(float, desc)
for i in prodByCatIDRDD.flatMap(lambda x:sorted(x[1],key=lambda l:-float(l.split(",")[4]))).take(5): print i

#16,2,Riddell Youth 360 Custom Football Helmet,,299.99,http://images.acmesports.sports/Riddell+Youth+360+Custom+Football+Helmet
#11,2,Fitness Gear 300 lb Olympic Weight Set,,209.99,http://images.acmesports.sports/Fitness+Gear+300+lb+Olympic+Weight+Set
#5,2,Riddell Youth Revolution Speed Custom Footbal,,199.99,http://images.acmesports.sports/Riddell+Youth+Revolution+Speed+Custom+Football+Helmet
#14,2,Quik Shade Summit SX170 10 FT. x 10 FT. Canop,,199.99,http://images.acmesports.sports/Quik+Shade+Summit+SX170+10+FT.+x+10+FT.+Canopy
#12,2,Under Armour Men's Highlight MC Alter Ego Fla,,139.99,http://images.acmesports.sports/Under+Armour+Men%27s+Highlight+MC+Alter+Ego+Flash+Football...

#rank function
#get top two products by price in each category
for i in prodByCatIDRDD.flatMap(lambda x:sorted(x[1],key=lambda l:-float(l.split(",")[4]))[:2]).take(10): print i

# following will fail
for i in prodByCatIDRDD.flatMap(lambda x:sorted(x[1],key=lambda l:-float(l.split(",")[4]))[:2]).take(100): print i
for i in sorted(prodByCatIDRDD.flatMap(lambda l:l[1]).map(lambda x:x.split(",")[4]).distinct().collect()): print i

# because the following record has a comma in product name
for i in prodByCatIDRDD.flatMap(lambda l:l[1]).filter(lambda x:x.split(",")[4]=="").collect(): print i
#685,31,TaylorMade SLDR Irons - (Steel) 4-PW, AW,,899.99,http://images.acmesports.sports/TaylorMade+SLDR+Irons+-+%28Steel%29+4-PW%2C+AW

<commit_msg>Update pyspark-ranking.txt<commit_after>pyspark -master local

productsRDD = sc.textFile("sqoop_import/products/")

# Products table desc from hive
#product_id              int
#product_category_id     int
#product_name            string
#product_description     string
#product_price           double
#product_image           string

prodByCatIDRDD = productsRDD.map(lambda x:(int(x.split(",")[1]),x)).groupByKey()
for i in prodByCatIDRDD.map(lambda x: list(x[1])).take(1)[0]: print i

# sort products by price(char)
for i in prodByCatIDRDD.flatMap(lambda x:sorted(x[1],key=lambda l:l.split(",")[4])).take(5): print i

#19,2,Nike Men's Fingertrap Max Training Shoe,,124.99,http://images.acmesports.sports/Nike+Men%27s+Fingertrap+Max+Training+Shoe
#2,2,Under Armour Men's Highlight MC Football Clea,,129.99,http://images.acmesports.sports/Under+Armour+Men%27s+Highlight+MC+Football+Cleat
#8,2,Nike Men's Vapor Carbon Elite TD Football Cle,,129.99,http://images.acmesports.sports/Nike+Men%27s+Vapor+Carbon+Elite+TD+Football+Cleat
#10,2,Under Armour Men's Highlight MC Football Clea,,129.99,http://images.acmesports.sports/Under+Armour+Men%27s+Highlight+MC+Football+Cleat
#17,2,Under Armour Men's Highlight MC Football Clea,,129.99,http://images.acmesports.sports/Under+Armour+Men%27s+Highlight+MC+Football+Cleat

# sort products by price(float, asc)
for i in prodByCatIDRDD.flatMap(lambda x:sorted(x[1],key=lambda l:float(l.split(",")[4]))).take(5): print i

#18,2,Reebok Men's Full Zip Training Jacket,,29.97,http://images.acmesports.sports/Reebok+Men%27s+Full+Zip+Training+Jacket
#22,2,Kijaro Dual Lock Chair,,29.99,http://images.acmesports.sports/Kijaro+Dual+Lock+Chair
#9,2,Nike Adult Vapor Jet 3.0 Receiver Gloves,,50.0,http://images.acmesports.sports/Nike+Adult+Vapor+Jet+3.0+Receiver+Gloves
#21,2,Under Armour Kids' Highlight RM Football Clea,,54.99,http://images.acmesports.sports/Under+Armour+Kids%27+Highlight+RM+Football+Cleat
#1,2,Quest Q64 10 FT. x 10 FT. Slant Leg Instant U,,59.98,http://images.acmesports.sports/Quest+Q64+10+FT.+x+10+FT.+Slant+Leg+Instant+Up+Canopy

# sort products by price(float, desc)
for i in prodByCatIDRDD.flatMap(lambda x:sorted(x[1],key=lambda l:-float(l.split(",")[4]))).take(5): print i

#16,2,Riddell Youth 360 Custom Football Helmet,,299.99,http://images.acmesports.sports/Riddell+Youth+360+Custom+Football+Helmet
#11,2,Fitness Gear 300 lb Olympic Weight Set,,209.99,http://images.acmesports.sports/Fitness+Gear+300+lb+Olympic+Weight+Set
#5,2,Riddell Youth Revolution Speed Custom Footbal,,199.99,http://images.acmesports.sports/Riddell+Youth+Revolution+Speed+Custom+Football+Helmet
#14,2,Quik Shade Summit SX170 10 FT. x 10 FT. Canop,,199.99,http://images.acmesports.sports/Quik+Shade+Summit+SX170+10+FT.+x+10+FT.+Canopy
#12,2,Under Armour Men's Highlight MC Alter Ego Fla,,139.99,http://images.acmesports.sports/Under+Armour+Men%27s+Highlight+MC+Alter+Ego+Flash+Football...

#rank function
#get top two products by price in each category
for i in prodByCatIDRDD.flatMap(lambda x:sorted(x[1],key=lambda l:-float(l.split(",")[4]))[:2]).take(10): print i

# following will fail
for i in prodByCatIDRDD.flatMap(lambda x:sorted(x[1],key=lambda l:-float(l.split(",")[4]))[:2]).take(100): print i
for i in sorted(prodByCatIDRDD.flatMap(lambda l:l[1]).map(lambda x:x.split(",")[4]).distinct().collect()): print i

# because the following record has a comma in product name
for i in prodByCatIDRDD.flatMap(lambda l:l[1]).filter(lambda x:x.split(",")[4]=="").collect(): print i
#685,31,TaylorMade SLDR Irons - (Steel) 4-PW, AW,,899.99,http://images.acmesports.sports/TaylorMade+SLDR+Irons+-+%28Steel%29+4-PW%2C+AW

# edit the data file in hue data browser and reload textfile
productsRDD = sc.textFile("sqoop_import/products/")
prodByCatIDRDD = productsRDD.map(lambda x:(int(x.split(",")[1]),x)).groupByKey()

for i in prodByCatIDRDD.flatMap(lambda l:l[1]).filter(lambda x:x.split(",")[4]=="").collect(): print i
# no records :)

# take top 2 products 
for i in prodByCatIDRDD.flatMap(lambda x:sorted(x[1],key=lambda l:-float(l.split(",")[4]))[:2]).take(100): print i

# samaple data
#1054,47,Spalding NBA 54" Glass Portable Basketball Ho,,699.99,http://images.acmesports.sports/Spalding+NBA+54%22+Glass+Portable+Basketball+Hoop
#1084,49,Lifetime Freestyle XL Stand-Up Paddle Board,,399.99,http://images.acmesports.sports/Lifetime+Freestyle+XL+Stand-Up+Paddle+Board
#1104,49,GoPro HERO3+ Black Edition Camera,,399.99,http://images.acmesports.sports/GoPro+HERO3%2B+Black+Edition+Camera

# take all the products with top 2 prices in each category
# key as category id and price 
prodByCatPriceRDD = productsRDD.map(lambda x:((int(x.split(",")[1]),float(x.split(",")[4])),x)).sortBy(lambda x: (x[0][0],-x[0][1]))
for i in prodByCatPriceRDD.take(10): print i

# take the key and 
prodCatPriceKeyRDD = prodByCatPriceRDD.keys().distinct().sortBy(lambda x: (x[0],-x[1])).groupByKey().map(lambda x:(x[0],list(x[1])[:2])).flatMapValues(lambda x:x)
for i in prodCatPriceKeyRDD.take(10): print i


{'fraction_non_alphanumeric': 0.16812933025404156, 'fraction_numerical': 0.056812933025404154, 'mean_word_length': 6.866485013623978, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2068519', 'n_tokens_mistral': 3653, 'n_tokens_neox': 3290, 'n_words': 583}