<jupyter_start><jupyter_text>Beer Sheva Data Files Analysis<jupyter_code>import json
import requests
from dataflows import Flow, printer, set_type, concatenate
HEADERS={
    'Pragma': 'no-cache',
    'Cache-Control': 'no-cache',
    'Upgrade-Insecure-Requests': '1',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate',
    'Accept-Language': 'en-US,en;q=0.9,he;q=0.8',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
}<jupyter_output><empty_output><jupyter_text>Open Jobs API<jupyter_code>URL = 'http://www.beer-sheva.muni.il/_layouts/GetjsonItems.aspx?List=e3ace9fe-6331-4b1a-af84-e3acc0041c71'
content = requests.get(URL, headers=HEADERS).text
data = json.loads(content.replace('\t',' '))
items = data['Root']['Items']['Item']
items[0]
processed = []
for item in items:
    processed.append(dict(
        (f['Caption'], f['Value'])
        for f in item['Fields']['Field']
    ))
processed[0]
print()
Flow(processed,
     concatenate({
         'title': ['שם מכרז'],
         'tender_id': ['מספר מכרז'],
         'claim_date': ['תאריך אחרון להגשה'],
         'update_date': ['תאריך עדכון'],
     }),
     set_type('claim_date', type='datetime', format='%d/%m/%Y %H:%M:%S'),
     set_type('update_date', type='date', format='%d/%m/%Y %H:%M:%S'),
     printer(tablefmt='html')
).process()<jupyter_output>
{'fraction_non_alphanumeric': 0.19959677419354838, 'fraction_numerical': 0.036290322580645164, 'mean_word_length': 4.5353159851301115, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '164864', 'license': 'MIT', 'n_tokens_mistral': 624, 'n_tokens_neox': 596, 'n_words': 94, 'path': 'BeerSheva/BeerSheva-Data-Analysis.ipynb', 'repo_name': 'hasadna/datacity-data-analysis', 'size': '1488'}