<commit_before>extern crate osmpbfreader;
use std::collections::HashMap;
use models::*;
use categorize::*;
use std;

// Way as represented in OpenStreetMap
struct Way {
    id: i64,
    nodes: Vec<i64>,
    properties: EdgeProperties,
}

struct Reader {
    nodes: HashMap<i64, Node>,
    ways: Vec<Way>,
}

impl Reader {
    fn new() -> Reader {
        Reader {
            nodes: HashMap::new(),
            ways: Vec::new(),
        }
    }

    fn count_nodes_uses(&mut self) {
        for way in &self.ways {
            for (i, node_id) in way.nodes.iter().enumerate() {
                if let Some(node) = self.nodes.get_mut(node_id) {
                    // Count double extremities nodes
                    if i == 0 || i == way.nodes.len() - 1 {
                        node.uses += 2;
                    } else {
                        node.uses += 1;
                    }
                } else {
                    panic!("Missing node, id: {}", node_id)
                }
            }
        }
    }


    fn split_way(&self, way: &Way) -> Vec<Edge> {
        let mut result = Vec::new();

        let mut source = 0;
        let mut geometry = Vec::new();
        for (i, &node_id) in way.nodes.iter().enumerate() {
            let node = self.nodes[&node_id];
            if i == 0 {
                source = node_id;
                geometry.push(node.coord);
            } else {
                geometry.push(node.coord);

                if node.uses > 1 {
                    result.push(Edge {
                        id: way.id,
                        source: source,
                        target: node_id,
                        geometry: geometry,
                        properties: way.properties,
                    });

                    source = node_id;
                    geometry = vec![node.coord];
                }
            }
        }
        result
    }

    fn read(&mut self, filename: &str) -> Result<(), String> {
        let path = std::path::Path::new(filename);
        let r = try!(std::fs::File::open(&path).map_err(|e| e.to_string()));
        let mut pbf = osmpbfreader::OsmPbfReader::new(r);
        for obj in pbf.iter() {
            match obj {
                osmpbfreader::OsmObj::Node(node) => {
                    let mut n = Node::new();
                    n.id = node.id;
                    n.coord = Coord {
                        lon: node.lon,
                        lat: node.lat,
                    };
                    self.nodes.insert(node.id, n);
                }
                osmpbfreader::OsmObj::Way(way) => {
                    let mut properties = EdgeProperties::new();
                    for (key, val) in way.tags {
                        properties.update(key, val);
                    }
                    properties.normalize();
                    if properties.accessible() {
                        self.ways.push(Way {
                            id: way.id,
                            nodes: way.nodes,
                            properties: properties,
                        })
                    }
                }
                osmpbfreader::OsmObj::Relation(_) => {}
            }
        }
        Ok(())
    }


    fn nodes(&self) -> Vec<Node> {
        self.nodes
            .iter()
            .map(|(_, node)| node)
            .filter(|node| node.uses > 1)
            .map(|n| n.clone())
            .collect()
    }

    fn edges(&self) -> Vec<Edge> {
        self.ways.iter().flat_map(|way| self.split_way(way)).collect()
    }
}

// Read all the nodes and ways of the osm.pbf file
pub fn read(filename: &str) -> Result<(Vec<Node>, Vec<Edge>), String> {
    let mut r = Reader::new();
    try!(r.read(filename));
    r.count_nodes_uses();
    Ok((r.nodes(), r.edges()))
}

#[test]
fn test_real_all() {<commit_msg>consume the nodes not to copy them all<commit_after>extern crate osmpbfreader;
use std::collections::HashMap;
use models::*;
use categorize::*;
use std;

// Way as represented in OpenStreetMap
struct Way {
    id: i64,
    nodes: Vec<i64>,
    properties: EdgeProperties,
}

struct Reader {
    nodes: HashMap<i64, Node>,
    ways: Vec<Way>,
}

impl Reader {
    fn new() -> Reader {
        Reader {
            nodes: HashMap::new(),
            ways: Vec::new(),
        }
    }

    fn count_nodes_uses(&mut self) {
        for way in &self.ways {
            for (i, node_id) in way.nodes.iter().enumerate() {
                if let Some(node) = self.nodes.get_mut(node_id) {
                    // Count double extremities nodes
                    if i == 0 || i == way.nodes.len() - 1 {
                        node.uses += 2;
                    } else {
                        node.uses += 1;
                    }
                } else {
                    panic!("Missing node, id: {}", node_id)
                }
            }
        }
    }


    fn split_way(&self, way: &Way) -> Vec<Edge> {
        let mut result = Vec::new();

        let mut source = 0;
        let mut geometry = Vec::new();
        for (i, &node_id) in way.nodes.iter().enumerate() {
            let node = self.nodes[&node_id];
            if i == 0 {
                source = node_id;
                geometry.push(node.coord);
            } else {
                geometry.push(node.coord);

                if node.uses > 1 {
                    result.push(Edge {
                        id: way.id,
                        source: source,
                        target: node_id,
                        geometry: geometry,
                        properties: way.properties,
                    });

                    source = node_id;
                    geometry = vec![node.coord];
                }
            }
        }
        result
    }

    fn read(&mut self, filename: &str) -> Result<(), String> {
        let path = std::path::Path::new(filename);
        let r = try!(std::fs::File::open(&path).map_err(|e| e.to_string()));
        let mut pbf = osmpbfreader::OsmPbfReader::new(r);
        for obj in pbf.iter() {
            match obj {
                osmpbfreader::OsmObj::Node(node) => {
                    let mut n = Node::new();
                    n.id = node.id;
                    n.coord = Coord {
                        lon: node.lon,
                        lat: node.lat,
                    };
                    self.nodes.insert(node.id, n);
                }
                osmpbfreader::OsmObj::Way(way) => {
                    let mut properties = EdgeProperties::new();
                    for (key, val) in way.tags {
                        properties.update(key, val);
                    }
                    properties.normalize();
                    if properties.accessible() {
                        self.ways.push(Way {
                            id: way.id,
                            nodes: way.nodes,
                            properties: properties,
                        })
                    }
                }
                osmpbfreader::OsmObj::Relation(_) => {}
            }
        }
        Ok(())
    }


    fn nodes(self) -> Vec<Node> {
        self.nodes
            .into_iter()
            .map(|(_, node)| node)
            .filter(|node| node.uses > 1)
            .collect()
    }

    fn edges(&self) -> Vec<Edge> {
        self.ways.iter().flat_map(|way| self.split_way(way)).collect()
    }
}

// Read all the nodes and ways of the osm.pbf file
pub fn read(filename: &str) -> Result<(Vec<Node>, Vec<Edge>), String> {
    let mut r = Reader::new();
    try!(r.read(filename));
    r.count_nodes_uses();
    let edges = r.edges();
    Ok((r.nodes(), edges)
}

#[test]
fn test_real_all() {
{'fraction_non_alphanumeric': 0.1295745506271822, 'fraction_numerical': 0.003620845726108884, 'mean_word_length': 1.1317530319735392, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4505945', 'n_tokens_mistral': 2239, 'n_tokens_neox': 2088, 'n_words': 486}