<commit_before>/* 
 * Copyright (C) 2016 Stanford University
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package net.ellitron.ldbcsnbimpls.interactive.neo4j.util;

/**
 * A utility for converting dataset files generated by the LDBC SNB Data
 * Generator[1] to the file format expected by the Neo4j import tool[2].
 * <p>
 * TODO:<br>
 * <ul>
 * <li>Add specific instructions.</li>
 * <li>Join language and e-mails into person node.</li>
 * </ul>
 * <p>
 * [1]: <EMAIL>@github.<EMAIL>:ldbc/ldbc_snb_datagen.git<br>
 * [2]: http://neo4j.com/docs/stable/import-tool.html<br>
 *
 * @author <NAME> (<EMAIL>)
 */
public class DataFormatConverter {

  /**
   * Represents all the types of nodes in the graph. Each node has an "ID
   * space" (see Neo4j Import Tool documentation) in which its
   * LDBC-SNB-Data-Generator-assigned ID is unique with every other node in
   * that space. For instance, comments and posts live in the "MESSAGE_ID"
   * space and all have unique IDs when considered together.
   *
   * Each node also has a "label". This is the label that's used to refer to
   * the node in Cypher queries. This label is overloaded in the sense that it
   * is also the string used to identify the dataset file containing these
   * nodes (e.g. person_0.0.csv)
   */
  private enum Node {

    COMMENT("MESSAGE_ID", "comment"),
    FORUM("FORUM_ID", "forum"),
    ORGANISATION("ORGANISATION_ID", "organisation"),
    PERSON("PERSON_ID", "person"),
    PLACE("PLACE_ID", "place"),
    POST("MESSAGE_ID", "post"),
    TAG("TAG_ID", "tag"),
    TAGCLASS("TAGCLASS_ID", "tagclass");

    private final String idSpace;
    private final String label;

    private Node(String idSpace, String label) {
      this.idSpace = idSpace;
      this.label = label;
    }

    public String getIdSpace() {
      return idSpace;
    }

    public String getLabel() {
      return label;
    }
  }

  /**
   * Represents all the types of relationships in the graph. Each relationship
   * has a "type" which is used to refer to relationships of that type in
   * Cypher queries. This "type" is overloaded in the sense that it is also
   * part of the filenames that contain relationships of this type (e.g.
   * person_isLocatedIn_place_0_0.csv).
   */
  private enum Relationship {

    CONTAINEROF("containerOf"),
    HASCREATOR("hasCreator"),
    HASINTEREST("hasInterest"),
    HASMEMBER("hasMember"),
    HASMODERATOR("hasModerator"),
    HASTAG("hasTag"),
    HASTYPE("hasType"),
    ISLOCATEDIN("isLocatedIn"),
    ISPARTOF("isPartOf"),
    ISSUBCLASSOF("isSubclassOf"),
    KNOWS("knows"),
    LIKES("likes"),
    REPLYOF("replyOf"),
    SPEAKS("speaks"),
    STUDYAT("studyAt"),
    WORKAT("workAt");

    private final String type;

    private Relationship(String type) {
      this.type = type;
    }
  }

  private static void printUsage() {
    System.out.println("Usage: DataFormatConverter inputDir outputDir");
    System.out.println("  inputDir  Directory containing SNB dataset files.");
    System.out.println("  outputDir Destination directory for output files.");
  }

  public static void main(String[] args) {
    if (args.length != 2) {
      printUsage();
      return;
    }

    String inputDir = args[0];
    String outputDir = args[1];
  }
}<commit_msg>Added initial implementation for nodes.<commit_after>/* 
 * Copyright (C) 2016 Stanford University
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package net.ellitron.ldbcsnbimpls.interactive.neo4j.util;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;

/**
 * A utility for converting dataset files generated by the LDBC SNB Data
 * Generator[1] to the file format expected by the Neo4j import tool[2].
 * <p>
 * TODO:<br>
 * <ul>
 * <li>Add specific instructions.</li>
 * <li>Join language and e-mails into person node.</li>
 * <li>More flexible file searching for multipart files.</li>
 * </ul>
 * <p>
 * [1]: <EMAIL>:ldbc/ldbc_snb_datagen.git<br>
 * [2]: http://neo4j.com/docs/stable/import-tool.html<br>
 *
 * @author <NAME> (<EMAIL>)
 */
public class DataFormatConverter {
  
  /**
   * Represents all the types of nodes in the graph and their various
   * properties.
   */
  private enum Node {

    COMMENT("MESSAGE_ID", "id", "Comment", "comment",
        new String[]{"creationDate:long", "locationIP:string",
          "browserUsed:string", "content:string", "length:int"}),
    FORUM("FORUM_ID", "id", "Forum", "forum",
        new String[]{"title:string", "creationDate:long"}),
    ORGANISATION("ORGANISATION_ID", "id", "Organisation", "organisation",
        new String[]{"type:string", "name:string", "url:string"}),
    PERSON("PERSON_ID", "id", "Person", "person",
        new String[]{"firstName:string", "lastName:string", "gender:string",
          "birthday:long", "creationDate:long", "locationIP:string",
          "browserUsed:string"}),
    PLACE("PLACE_ID", "id", "Place", "place",
        new String[]{"name:string", "url:string", "type:string"}),
    POST("MESSAGE_ID", "id", "Post", "post",
        new String[]{"imageFile:string", "creationDate:long",
          "locationIP:string", "browserUsed:string", "language:string",
          "content:string", "length:int"}),
    TAG("TAG_ID", "id", "Tag", "tag",
        new String[]{"name:string", "url:string"}),
    TAGCLASS("TAGCLASS_ID", "id", "TagClass", "tagclass",
        new String[]{"name:string", "url:string"});

    /*
     * "ID space" for the node (see Neo4j Import Tool documentation). IDs
     * within an ID space must be unique. Note that Comments and Posts both
     * belong to the ID space for messages. This is because their IDs are
     * unique together in this space, and the benchmark sometimes requires
     * quering a "message" with a specific ID, which could be either a post or
     * a comment.
     */
    private final String neoIdSpace;

    /*
     * The property key used to refer to the id in Cypher queries.
     */
    private final String neoIdPropKey;

    /*
     * The label given to these nodes. We use the same labels here as used in
     * the Cypher queries for Neo4j listed in the appendix of the LDBC SNB
     * specification v0.2.2.
     */
    private final String neoLabel;

    /*
     * This is the name used in fileNames for the node as output by the LDBC
     * SNB Data Generator.
     */
    private final String fileLabel;

    /*
     * Ordered array of the properties for this node and their types, in the
     * order they appear in the LDBC SNB Data Generator dataset files.
     */
    private final String[] props;

    private Node(String neoIdSpace, String neoIdPropKey,
        String neoLabel, String fileLabel, String[] props) {
      this.neoIdSpace = neoIdSpace;
      this.neoIdPropKey = neoIdPropKey;
      this.neoLabel = neoLabel;
      this.fileLabel = fileLabel;
      this.props = props;
    }

    public String getNeoIdSpace() {
      return neoIdSpace;
    }

    public String getNeoIdPropKey() {
      return neoIdPropKey;
    }

    public String getNeoLabel() {
      return neoLabel;
    }

    public String getFileLabel() {
      return fileLabel;
    }
    
    public String[] getProps() {
      return props;
    }
  }

  /**
   * Represents all the types of relationships in the graph. Each relationship
   * has a "type" which is used to refer to relationships of that type in
   * Cypher queries. This "type" is overloaded in the sense that it is also
   * part of the filenames that contain relationships of this type (e.g.
   * person_isLocatedIn_place_0_0.csv).
   */
  private enum Relationship {

    CONTAINEROF("containerOf"),
    HASCREATOR("hasCreator"),
    HASINTEREST("hasInterest"),
    HASMEMBER("hasMember"),
    HASMODERATOR("hasModerator"),
    HASTAG("hasTag"),
    HASTYPE("hasType"),
    ISLOCATEDIN("isLocatedIn"),
    ISPARTOF("isPartOf"),
    ISSUBCLASSOF("isSubclassOf"),
    KNOWS("knows"),
    LIKES("likes"),
    REPLYOF("replyOf"),
    SPEAKS("speaks"),
    STUDYAT("studyAt"),
    WORKAT("workAt");

    private final String type;

    private Relationship(String type) {
      this.type = type;
    }
  }

  private static void printUsage() {
    System.out.println("Usage: DataFormatConverter inputDir outputDir");
    System.out.println("  inputDir  Directory containing SNB dataset files.");
    System.out.println("  outputDir Destination directory for output files.");
  }

  public static void main(String[] args) throws FileNotFoundException, IOException {
    if (args.length != 2) {
      printUsage();
      return;
    }

    String inputDir = args[0];
    String outputDir = args[1];

    // Start with the nodes.
    for (Node node : Node.values()) {
      String fileName = node.getFileLabel() + "_0_0.csv";

      Path path = Paths.get(inputDir + "/" + fileName);
      BufferedReader inFile =
          Files.newBufferedReader(path, StandardCharsets.UTF_8);

      path = Paths.get(outputDir + "/" + fileName);
      BufferedWriter outFile =
          Files.newBufferedWriter(path, StandardCharsets.UTF_8);

      String line;
      boolean firstLine = true;
      while ((line = inFile.readLine()) != null) {
        if (firstLine) {
          // Replace the headers with those expected by the Neo4j Import Tool.
          outFile.append(String.format(
              "%s:ID(%s)", node.getNeoIdPropKey(), node.getNeoIdSpace()));
          
          for (String property : node.getProps()) {
            outFile.append("|" + property);
          }
          
          outFile.append("\n");
          
          firstLine = false;
        } else {
          // The rest of the file is the same.
          outFile.append(line);
          outFile.append("\n");
        }
      }
      
      inFile.close();
      outFile.close();
    }
  }
}
{'fraction_non_alphanumeric': 0.1169016598250937, 'fraction_numerical': 0.004194181688381224, 'mean_word_length': 3.05756698044895, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1122430', 'n_tokens_mistral': 3445, 'n_tokens_neox': 3301, 'n_words': 1093}