<commit_before>"""Test script with parsing functions

This script can run on a regular python console, it is just a set of 
helper functions to extract resources, selectors and operators from urls
according to the rules described in 
http://developer.ning.com/notes/REST_API_Request_URL_Syntax

The idea is to use this in the main project as it matures.
"""

import urllib2
import re

# some possible urls
urls = (
    "http://networkcreators.ning.com/xn/atom/1.0",
    "http://networkcreators.ning.com/xn/atom/1.0/content()",
    "http://developer.ning.com/xn/atom/1.0/content?order=published@D",
    "http://networkcreators.ning.com/xn/atom/1.0/content(type='User'&author='david')",
    "http://brooklynartproject.ning.com/xn/atom/1.0/content(type='Photo')?order=my.viewCount@D&from=0&to=10",
    "http://networkcreators.ning.com/xn/atom/1.0/content(type='Topic'&my.xg_forum_commentCount>1)?order=my.xg_forum_commentCount@D&from=0&to=5",
    "http://jetsetshow.ning.com/xn/atom/1.0/content(type='Topic')?order=published@A&from=0&to=5",
    "http://developer.ning.com/xn/atom/1.0/content/rollup(field='type')",
    "http://networkcreators.ning.com/xn/atom/1.0/profile(id='david')",
    "http://brooklynartproject.ning.com/xn/atom/1.0/tag(value='brooklynbattle')/content(type='Photo')?order=my.ratingCount@D&from=0&to=10",
)<commit_msg>minor, adding shebang and encoding to the test script<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""Test script with parsing functions

This script can run on a regular python console, it is just a set of 
helper functions to extract resources, selectors and operators from urls
according to the rules described in 
http://developer.ning.com/notes/REST_API_Request_URL_Syntax

The idea is to use this in the main project as it matures.
"""

import urllib2
import re

# some possible urls
urls = (
    "http://networkcreators.ning.com/xn/atom/1.0",
    "http://networkcreators.ning.com/xn/atom/1.0/content()",
    "http://developer.ning.com/xn/atom/1.0/content?order=published@D",
    "http://networkcreators.ning.com/xn/atom/1.0/content(type='User'&author='david')",
    "http://brooklynartproject.ning.com/xn/atom/1.0/content(type='Photo')?order=my.viewCount@D&from=0&to=10",
    "http://networkcreators.ning.com/xn/atom/1.0/content(type='Topic'&my.xg_forum_commentCount>1)?order=my.xg_forum_commentCount@D&from=0&to=5",
    "http://jetsetshow.ning.com/xn/atom/1.0/content(type='Topic')?order=published@A&from=0&to=5",
    "http://developer.ning.com/xn/atom/1.0/content/rollup(field='type')",
    "http://networkcreators.ning.com/xn/atom/1.0/profile(id='david')",
    "http://brooklynartproject.ning.com/xn/atom/1.0/tag(value='brooklynbattle')/content(type='Photo')?order=my.ratingCount@D&from=0&to=10",
)
{'fraction_non_alphanumeric': 0.17550872093023256, 'fraction_numerical': 0.023619186046511628, 'mean_word_length': 5.010917030567685, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1024514', 'n_tokens_mistral': 1050, 'n_tokens_neox': 1022, 'n_words': 148}