<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import lxml.etree, lxml.html
import re

url = "http://redis.io"
output = "output.txt"

f = open(output, "w");

tree = lxml.html.parse("download/raw.dat").getroot()
commands = tree.find_class("command")

data = {}

for command in commands:

    for row in command.findall('a'):
        command_url = "%s%s" % (url, row.get('href'))
        
        for sibling in command.itersiblings():
            usage = ""
            
            for command_args in command.findall('span'):
                usage = "Usage: %s %s" % (row.text, command_args.text.replace(' ', '').replace('\n', ' ').strip())

            summary = "%s<br>%s" % (sibling.text, usage)

            data[command_url] = (row.text, summary)

for command_url in data.keys():
    command, summary = data[command_url]
    summary = unicode(summary).encode("utf-8")
    
    f.write("\t".join([str(command),      # title
                    "",                # namespace
                    command_url,               # url
                    summary,       # description
                    "",                # synopsis
                    "",                # details
                    "",                # type
                    ""                 # lang
                   ])
           )
    f.write("\n")
f.close()<commit_msg>redis_commands: Use a synopsis for usage examples<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import lxml.etree, lxml.html
import re

url = "http://redis.io"
output = "output.txt"

f = open(output, "w");

tree = lxml.html.parse("download/raw.dat").getroot()
commands = tree.find_class("command")

data = {}

for command in commands:

    for row in command.findall('a'):
        command_url = "%s%s" % (url, row.get('href'))
        
        for sibling in command.itersiblings():
            usage = ""
            
            for command_args in command.findall('span'):
                usage = "%s %s" % (row.text, command_args.text.replace(' ', '').replace('\n', ' ').strip())

            summary = "%s" % (sibling.text)

            data[command_url] = (row.text, summary, usage)

for command_url in data.keys():
    command, summary, usage = data[command_url]
    summary = unicode(summary).encode("utf-8")
    usage = unicode(usage).encode("utf-8")
    
    f.write("\t".join([str(command),      # title
                    "",                # namespace
                    command_url,               # url
                    summary,       # description
                    usage,                # synopsis
                    "",                # details
                    "",                # type
                    ""                 # lang
                   ])
           )
    f.write("\n")
f.close()
{'fraction_non_alphanumeric': 0.15078796561604585, 'fraction_numerical': 0.0017908309455587394, 'mean_word_length': 1.4414335664335665, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3707699', 'n_tokens_mistral': 817, 'n_tokens_neox': 789, 'n_words': 164}