<commit_before>#!/usr/bin/python3
# OMDB_api_scrape.py - parses a movie and year from the command line, grabs the
#                      JSON and saves a copy for later

import json, requests, sys, os

URL_BASE = 'http://www.omdbapi.com/?'

if len(sys.argv) > 1:
	# Get address from command line.
    mTitle = '+'.join(sys.argv[1:-1])
    mYear = sys.argv[-1]
    print(mTitle)
    print(mYear)
else:
    print("Usage: OMDB_api_scrape.py <Movie Title> <Year>")
    sys.exit(1)

# Craft the URL
url = URL_BASE + 't=' + mTitle + '&y=' + mYear + '&plot=full&r=json'

# Try to get the url
try:
	response = requests.get(url)
	response.raise_for_status()
except requests.exceptions.RequestException as err:
	print(err)
	sys.exit(1)

theJSON = json.loads(response.text)

# Save the JSON file
with open((os.path.join(os.getcwd(), (mTitle + '_' + mYear + '.json'))), 'w') as outfile:
    json.dump(theJSON, outfile)<commit_msg>Convert OMDB scrapper to grab xml<commit_after>#!/usr/bin/python3
# OMDB_api_scrape.py - parses a movie and year from the command line, grabs the
#                      xml and saves a copy for later

import requests, sys, os
import lxml.etree

URL_BASE = 'http://www.omdbapi.com/?'

if len(sys.argv) > 1:
	# Get address from command line.
    mTitle = '+'.join(sys.argv[1:-1])
    mYear = sys.argv[-1]
    print(mTitle)
    print(mYear)
else:
    print("Usage: OMDB_api_scrape.py <Movie Title> <Year>")
    sys.exit(1)

# Craft the URL
url = URL_BASE + 't=' + mTitle + '&y=' + mYear + '&plot=full&r=xml'

# Try to get the url
try:
	response = requests.get(url)
	response.raise_for_status()
except requests.exceptions.RequestException as err:
	print(err)
	sys.exit(1)

# Save the XML file
with open((os.path.join(os.getcwd(), (mTitle + '_' + mYear + '.xml'))), 'wb') as outfile:
    outfile.write(response.text)
{'fraction_non_alphanumeric': 0.15144887916894478, 'fraction_numerical': 0.007654455986878075, 'mean_word_length': 3.485294117647059, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '610970', 'n_tokens_mistral': 698, 'n_tokens_neox': 683, 'n_words': 180}