<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# Turns a twitter thread URL into a thethreaderapp.com URL for easier reading

import requests
import argparse
import sys, os

# Lambda function to "Clear" the terminal
clear = lambda: os.system('cls' if os.name == 'nt' else 'clear')

parser = argparse.ArgumentParser(description='Get the thread ID from a twitter URL')
parser.add_argument('url', help='the URL of the thread')
args = parser.parse_args()

url = args.url
valid_twitter_domains = ['twitter.com', 'mobile.twitter.com', 't.co']

def format_url(url):
    # remove www. from the URL
    if 'www.' in url:
        url = url.replace('www.', '')
    
    # add https:// if it's missing
    if not url.startswith('https://') and not url.startswith('http://'):
        url = 'https://' + url
    return url

url = (format_url(url))

# Make a HEAD request to the URL to check if valid/still exists
def check_url(url):
    try:
        r = requests.head(url)
        if r.status_code == 200:
            return True
        else:
            return False
    except requests.exceptions.RequestException as e:
        print(e)
        sys.exit(1)

# get the thread ID from a twitter URL
def get_thread_id(url):
    if check_url(url):
        if not any(domain in url for domain in valid_twitter_domains):
            print('Invalid URL')
        else:
            # sanitize the URL of query strings
            url = url.split('?')[0]
            return url.split('/')[-1]

thread_id = get_thread_id(url)


# Generating a "The Thread App" URL
def threader_app(thread_id):
    threader_domain = "https://threadreaderapp.com/thread/"
    threader_url = threader_domain + thread_id + ".html"
    print(threader_url)


if __name__ == '__main__':
    clear()
    threader_app(thread_id)<commit_msg>	modified:   threader_app.py<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# Turns a twitter thread URL into a thethreaderapp.com URL for easier reading

import requests
import argparse
import sys

parser = argparse.ArgumentParser(description='Get the thread ID from a twitter URL')
parser.add_argument('url', help='the URL of the thread')
args = parser.parse_args()

url = args.url
valid_twitter_domains = ['twitter.com', 'mobile.twitter.com', 't.co']

def format_url(url):
    # remove www. from the URL
    if 'www.' in url:
        url = url.replace('www.', '')
    
    # add https:// if it's missing
    if not url.startswith('https://') and not url.startswith('http://'):
        url = 'https://' + url
    return url

url = format_url(url)

# Make a HEAD request to the URL to check if valid/still exists
def check_url(url):
    try:
        r = requests.head(url)
        if r.status_code == 200:
            return True
        else:
            return False
    except requests.exceptions.RequestException as e:
        print(e)
        sys.exit(1)

# get the thread ID from a twitter URL
def get_thread_id(url):
    if check_url(url):
        if not any(domain in url for domain in valid_twitter_domains):
            print('Invalid URL')
        else:
            # sanitize the URL of query strings
            url = url.split('?')[0]
            return url.split('/')[-1]

thread_id = get_thread_id(url)

# Generating a "The Thread App" URL
def threader_app(thread_id):
    threader_domain = "https://threadreaderapp.com/thread/"
    threader_url = threader_domain + thread_id + ".html"
    print(threader_url)


if __name__ == '__main__':
    threader_app(thread_id)
{'fraction_non_alphanumeric': 0.11922418710781517, 'fraction_numerical': 0.004563605248146035, 'mean_word_length': 2.799566630552546, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 8, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '605105', 'n_tokens_mistral': 1123, 'n_tokens_neox': 1094, 'n_words': 341}