<commit_before>//! A simple application that downloads the data dictionary and creates code or
//! data to reproduce it in the core library.
//!
//! ### How to use
//!
//! Simply run the application. It will automatically retrieve the dictionary
//! from the DCMTK data dictionary and store the result in "entries.rs".
//! Future versions will enable different kinds of outputs.
//!
//! Please use the `--help` flag for the full usage information.

use clap::{App, Arg};
use serde::Serialize;
use regex::Regex;

use heck::ShoutySnakeCase;
use std::{fs::{create_dir_all, File}, io::BufWriter};
use std::io::{BufRead, BufReader, Write};
use std::path::Path;

/// url to DCMTK dic file
const DEFAULT_LOCATION: &str = "https://raw.githubusercontent.com/DCMTK/dcmtk/master/dcmdata/data/dicom.dic";

fn main() {
    let matches = App::new("DICOM Dictionary Builder")
        .version("0.1.0")
        .arg(
            Arg::with_name("FROM")
                .default_value(DEFAULT_LOCATION)
                .help("Where to fetch the dictionary from"),
        )
        .arg(
            Arg::with_name("no-retired")
                .long("no-retired")
                .help("Whether to ignore retired tags")
                .takes_value(false),
        )
        .arg(
            Arg::with_name("OUTPUT")
                .short("o")
                .help("The path to the output file")
                .default_value("tags.rs")
                .takes_value(true),
        )
        .get_matches();

    let ignore_retired = matches.is_present("no-retired");

    let src = matches.value_of("FROM").unwrap();

    let dst = Path::new(matches.value_of("OUTPUT").unwrap());

    if src.starts_with("http:") || src.starts_with("https:") {
        // read from URL
        println!("Downloading DICOM dictionary ...");
        let mut resp = reqwest::get(src).unwrap();
        let mut data = vec![];
        resp.copy_to(&mut data).unwrap();

        let preamble = data.split(|&b| b == b'\n')
            .filter_map(|l| std::str::from_utf8(l).ok())
            .find(|l| l.contains("Copyright"))
            .unwrap_or("");
        let preamble = format!(
            "Adapted from the DCMTK project.\nURL: {}\n{}",
            src,
            preamble,
        );

        let entries = parse_entries(&*data).unwrap();
        println!("Writing to file ...");
        to_code_file(dst, entries, !ignore_retired, &preamble).expect("Failed to write file");
    } else {
        // read from File
        let file = File::open(src).unwrap();
        let entries = parse_entries(BufReader::new(file)).unwrap();
        println!("Writing to file ...");
        to_code_file(dst, entries, !ignore_retired, "").expect("Failed to write file");<commit_msg>[dictionary-builder] add link to license in preamble<commit_after>//! A simple application that downloads the data dictionary and creates code or
//! data to reproduce it in the core library.
//!
//! ### How to use
//!
//! Simply run the application. It will automatically retrieve the dictionary
//! from the DCMTK data dictionary and store the result in "entries.rs".
//! Future versions will enable different kinds of outputs.
//!
//! Please use the `--help` flag for the full usage information.

use clap::{App, Arg};
use serde::Serialize;
use regex::Regex;

use heck::ShoutySnakeCase;
use std::{fs::{create_dir_all, File}, io::BufWriter};
use std::io::{BufRead, BufReader, Write};
use std::path::Path;

/// url to DCMTK dic file
const DEFAULT_LOCATION: &str = "https://raw.githubusercontent.com/DCMTK/dcmtk/master/dcmdata/data/dicom.dic";

fn main() {
    let matches = App::new("DICOM Dictionary Builder")
        .version("0.1.0")
        .arg(
            Arg::with_name("FROM")
                .default_value(DEFAULT_LOCATION)
                .help("Where to fetch the dictionary from"),
        )
        .arg(
            Arg::with_name("no-retired")
                .long("no-retired")
                .help("Whether to ignore retired tags")
                .takes_value(false),
        )
        .arg(
            Arg::with_name("OUTPUT")
                .short("o")
                .help("The path to the output file")
                .default_value("tags.rs")
                .takes_value(true),
        )
        .get_matches();

    let ignore_retired = matches.is_present("no-retired");

    let src = matches.value_of("FROM").unwrap();

    let dst = Path::new(matches.value_of("OUTPUT").unwrap());

    if src.starts_with("http:") || src.starts_with("https:") {
        // read from URL
        println!("Downloading DICOM dictionary ...");
        let mut resp = reqwest::get(src).unwrap();
        let mut data = vec![];
        resp.copy_to(&mut data).unwrap();

        let preamble = data.split(|&b| b == b'\n')
            .filter_map(|l| std::str::from_utf8(l).ok())
            .find(|l| l.contains("Copyright"))
            .unwrap_or("");
        let preamble = format!(
            "Adapted from the DCMTK project.\nURL: {}\nLicense: {}\n{}",
            src,
            "https://github.com/DCMTK/dcmtk/blob/master/COPYRIGHT",
            preamble,
        );

        let entries = parse_entries(&*data).unwrap();
        println!("Writing to file ...");
        to_code_file(dst, entries, !ignore_retired, &preamble).expect("Failed to write file");
    } else {
        // read from File
        let file = File::open(src).unwrap();
        let entries = parse_entries(BufReader::new(file)).unwrap();
        println!("Writing to file ...");
        to_code_file(dst, entries, !ignore_retired, "").expect("Failed to write file");
{'fraction_non_alphanumeric': 0.15394220478059223, 'fraction_numerical': 0.0014270424545130217, 'mean_word_length': 2.3315508021390374, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '495557', 'n_tokens_mistral': 1704, 'n_tokens_neox': 1653, 'n_words': 429}