<commit_before>User-agent: *
Sitemap: http://courtlistener.com/sitemap.xml

# Added November 20, 2010
Disallow: /x/

# Added May 21, 2010
Disallow: /ca5/Fenghui-Fan-v.-Vicki-Brewer,-et-al/
Disallow: /ca5/Fenghui-Fan-v.-Vicki-Brewer/
Disallow: /pdf/2010/05/06/Fenghui_Fan_v._Vicki_Brewer_et_al.pdf
Disallow: /ca5/Kpt/fenghui-fan-v-vicki-brewer/

# Added May 24, 2010
Disallow: /ca2/Yero-v.-Holder/
Disallow: /pdf/2010/04/24/Yero_v._Holder.pdf
Disallow: /ca2/2gP/yero-v-holder/

# Added July 12, 2010
Disallow: /ca10/Cedrins-v.-USCIS/
Disallow: /pdf/2010/06/23/Cedrins_v._USCIS.pdf
Disallow: /ca10/LmL/cedrins-v-uscis/
Disallow: /ca10/Cedrins-v.-Central-New-Mexico-Community-College/
Disallow: /pdf/2010/06/23/Cedrins_v._Central_New_Mexico_Comm_Colleg.pdf
Disallow: /ca10/LmS/cedrins-v-central-new-mexico-community-college/

# Added July 20, 2010
Disallow: /ca11/Steven-M.-Bircoll-v.-Miami_Dade-County/
Disallow: /pdf/2010/04/26/Steven_M._Bircoll_v._Miami-Dade_County.pdf
Disallow: /ca11/p5t/steven-m-bircoll-v-miami-dade-county/

# Added August 9, 2010
Disallow: /ca5/United-States-v.-Willia-Allen/
Disallow: /pdf/2010/04/24/U.S._v._Allen.pdf
Disallow: /ca5/3y1/united-states-v-willia-allen/

# Added August 24, 2010
Disallow: /ca3/Chafik-Sebki-v.-Atty-Gen-USA/
Disallow: /pdf/2010/04/23/Chafik_Sebki_v._Atty_Gen_USA.pdf
Disallow: /ca3/GZ/chafik-sebki-v-atty-gen-usa/

# Added October 7, 2010
Disallow: /ca2/LXw/united-states-v-ahmad-kanan/
Disallow: /pdf/2010/07/21/United_States_v._Ahmad_Kanan.pdf
Disallow: /ca2/MbE/united-states-v-ahmad-kanan/
Disallow: /pdf/2010/08/02/United_States_v._Ahmad_Kanan_1.pdf 
Disallow: /pdf/2010/08/02/United_States_v._Ahmad_Kanan.pdf 
Disallow: /ca2/United-States-v.-Ahmad-Kanan/
Disallow: /ca2/United-States-v.-Ahmad-Kanan,/ 

# Added November 13, 2010
Disallow: /ca11/eXX/german-dario-rojas-iriarte-v-us-atty-gen/
Disallow: /pdf/2006/01/30/German_Dario_Rojas-Iriarte_v._U.S._Atty._Gen..pdf

# Added November 20, 2010
Disallow: /ca5/k6u/love-et-al-v-dell-inc-et-al/
Disallow: /pdf/2008/12/02/Love_et_al_v._Dell_Inc_et_al.pdf

# Added December 1, 2010
Disallow: /cafc/Vde/delapenia-v-merit-systems-protection-board/
Disallow: /pdf/2010/11/09/Delapenia_v._Merit_Systems_Protection_Board.pdf
Disallow: /cafc/M5S/delapenia-v-mspb/  
Disallow: /pdf/2010/07/27/Delapenia_v._MSPB.pdf

# Added December 15, 2010
Disallow: /ca5/aLj/united-states-v-collman/
Disallow: /pdf/2003/10/21/USA_v._Collman.pdf
Disallow: /ca5/aLj/usa-v-collman/<commit_msg>New additions to robots.txt<commit_after>User-agent: *
Sitemap: http://courtlistener.com/sitemap.xml

# Added November 20, 2010
Disallow: /x/

# Added May 21, 2010
Disallow: /ca5/Fenghui-Fan-v.-Vicki-Brewer,-et-al/
Disallow: /ca5/Fenghui-Fan-v.-Vicki-Brewer/
Disallow: /pdf/2010/05/06/Fenghui_Fan_v._Vicki_Brewer_et_al.pdf
Disallow: /ca5/Kpt/fenghui-fan-v-vicki-brewer/

# Added May 24, 2010
Disallow: /ca2/Yero-v.-Holder/
Disallow: /pdf/2010/04/24/Yero_v._Holder.pdf
Disallow: /ca2/2gP/yero-v-holder/

# Added July 12, 2010
Disallow: /ca10/Cedrins-v.-USCIS/
Disallow: /pdf/2010/06/23/Cedrins_v._USCIS.pdf
Disallow: /ca10/LmL/cedrins-v-uscis/
Disallow: /ca10/Cedrins-v.-Central-New-Mexico-Community-College/
Disallow: /pdf/2010/06/23/Cedrins_v._Central_New_Mexico_Comm_Colleg.pdf
Disallow: /ca10/LmS/cedrins-v-central-new-mexico-community-college/

# Added July 20, 2010
Disallow: /ca11/Steven-M.-Bircoll-v.-Miami_Dade-County/
Disallow: /pdf/2010/04/26/Steven_M._Bircoll_v._Miami-Dade_County.pdf
Disallow: /ca11/p5t/steven-m-bircoll-v-miami-dade-county/

# Added August 9, 2010
Disallow: /ca5/United-States-v.-Willia-Allen/
Disallow: /pdf/2010/04/24/U.S._v._Allen.pdf
Disallow: /ca5/3y1/united-states-v-willia-allen/

# Added August 24, 2010
Disallow: /ca3/Chafik-Sebki-v.-Atty-Gen-USA/
Disallow: /pdf/2010/04/23/Chafik_Sebki_v._Atty_Gen_USA.pdf
Disallow: /ca3/GZ/chafik-sebki-v-atty-gen-usa/

# Added October 7, 2010
Disallow: /ca2/LXw/united-states-v-ahmad-kanan/
Disallow: /pdf/2010/07/21/United_States_v._Ahmad_Kanan.pdf
Disallow: /ca2/MbE/united-states-v-ahmad-kanan/
Disallow: /pdf/2010/08/02/United_States_v._Ahmad_Kanan_1.pdf 
Disallow: /pdf/2010/08/02/United_States_v._Ahmad_Kanan.pdf 
Disallow: /ca2/United-States-v.-Ahmad-Kanan/
Disallow: /ca2/United-States-v.-Ahmad-Kanan,/ 

# Added November 13, 2010
Disallow: /ca11/eXX/german-dario-rojas-iriarte-v-us-atty-gen/
Disallow: /pdf/2006/01/30/German_Dario_Rojas-Iriarte_v._U.S._Atty._Gen..pdf

# Added November 20, 2010
Disallow: /ca5/k6u/love-et-al-v-dell-inc-et-al/
Disallow: /pdf/2008/12/02/Love_et_al_v._Dell_Inc_et_al.pdf

# Added December 1, 2010
Disallow: /cafc/Vde/delapenia-v-merit-systems-protection-board/
Disallow: /pdf/2010/11/09/Delapenia_v._Merit_Systems_Protection_Board.pdf
Disallow: /cafc/M5S/delapenia-v-mspb/  
Disallow: /pdf/2010/07/27/Delapenia_v._MSPB.pdf

# Added December 15, 2010
Disallow: /ca5/aLj/united-states-v-collman/
Disallow: /pdf/2003/10/21/USA_v._Collman.pdf
Disallow: /ca5/aLj/usa-v-collman/

# Added January 26, 2011
Disallow: /ca8/M88/claudia-fercello-v-county-of-ramsey/
Disallow: /pdf/2010/07/29/Claudia_Fercello_v._County_of_Ramsey.pdf
{'fraction_non_alphanumeric': 0.16147974163241338, 'fraction_numerical': 0.091798786455275, 'mean_word_length': 4.054401582591494, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6670197', 'n_tokens_mistral': 2827, 'n_tokens_neox': 2379, 'n_words': 276}