<commit_before>package apoc.gephi;

import org.neo4j.procedure.Description;
import apoc.graph.Graphs;
import apoc.result.ProgressInfo;
import apoc.util.JsonUtil;
import apoc.util.UrlResolver;
import apoc.util.Util;
import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.PropertyContainer;
import org.neo4j.graphdb.Relationship;
import org.neo4j.procedure.Name;
import org.neo4j.procedure.Procedure;

import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.ArrayList;
import java.util.Map;
import java.util.List;
import java.util.Arrays;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static apoc.util.MapUtil.map;

/**
 * @author mh
 * @since 29.05.16
 */
// https://github.com/gephi/gephi/wiki/GraphStreaming#Gephi_as_Master
// https://marketplace.gephi.org/plugin/graph-streaming/
public class Gephi {

    public static final int WIDTH = 1000;
    public static final int HEIGHT = 1000;

    private String getGephiUrl(String hostOrKey) {
        return new UrlResolver("http", "localhost", 8080).getUrl("gephi", hostOrKey);
    }

    public static double doubleValue(PropertyContainer pc, String prop, Number defaultValue) {
       return Util.toDouble(pc.getProperty(prop, defaultValue));

    }

    public static final String[] CAPTIONS = new String[]{"name", "title", "label"};
    public static final List<String> RESERVED = Arrays.asList("label", "TYPE", "id", "source", "target", "weight", "directed"); 
    // http://127.0.0.1:8080/workspace0?operation=updateGraph
    // TODO configure property-filters or transfer all properties
    @Procedure
    @Description("apoc.gephi.add(url-or-key, workspace, data, weightproperty, ['exportproperty']) | streams passed in data to Gephi")
    public Stream<ProgressInfo> add(@Name("urlOrKey") String keyOrUrl, @Name("workspace") String workspace, @Name("data") Object data,@Name(value = "weightproperty",defaultValue = "null") String weightproperty,@Name(value = "exportproperties",defaultValue = "[]") List<String> exportproperties) {
        if (workspace == null) workspace = "workspace0";
        String url = getGephiUrl(keyOrUrl)+"/"+Util.encodeUrlComponent(workspace)+"?operation=updateGraph";
        long start = System.currentTimeMillis();
        HashSet<Node> nodes = new HashSet<>(1000);
        HashSet<Relationship> rels = new HashSet<>(10000);
        List<String> propertyNames = new ArrayList<>(exportproperties);
        propertyNames.removeAll(RESERVED);
        if (Graphs.extract(data, nodes, rels)) {
            String payload = toGephiStreaming(nodes, rels, weightproperty, propertyNames.toArray(new String[propertyNames.size()]));
            JsonUtil.loadJson(url,map("method","POST"), payload).count();
            return Stream.of(new ProgressInfo(url,"graph","gephi").update(nodes.size(),rels.size(),nodes.size()).done(start));
        }
        return Stream.empty();
    }<commit_msg>Add charset=utf8 to Gephi streaming<commit_after>package apoc.gephi;

import org.neo4j.procedure.Description;
import apoc.graph.Graphs;
import apoc.result.ProgressInfo;
import apoc.util.JsonUtil;
import apoc.util.UrlResolver;
import apoc.util.Util;
import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.PropertyContainer;
import org.neo4j.graphdb.Relationship;
import org.neo4j.procedure.Name;
import org.neo4j.procedure.Procedure;

import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.ArrayList;
import java.util.Map;
import java.util.List;
import java.util.Arrays;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static apoc.util.MapUtil.map;

/**
 * @author mh
 * @since 29.05.16
 */
// https://github.com/gephi/gephi/wiki/GraphStreaming#Gephi_as_Master
// https://marketplace.gephi.org/plugin/graph-streaming/
public class Gephi {

    public static final int WIDTH = 1000;
    public static final int HEIGHT = 1000;

    private String getGephiUrl(String hostOrKey) {
        return new UrlResolver("http", "localhost", 8080).getUrl("gephi", hostOrKey);
    }

    public static double doubleValue(PropertyContainer pc, String prop, Number defaultValue) {
       return Util.toDouble(pc.getProperty(prop, defaultValue));

    }

    public static final String[] CAPTIONS = new String[]{"name", "title", "label"};
    public static final List<String> RESERVED = Arrays.asList("label", "TYPE", "id", "source", "target", "weight", "directed"); 
    // http://127.0.0.1:8080/workspace0?operation=updateGraph
    // TODO configure property-filters or transfer all properties
    @Procedure
    @Description("apoc.gephi.add(url-or-key, workspace, data, weightproperty, ['exportproperty']) | streams passed in data to Gephi")
    public Stream<ProgressInfo> add(@Name("urlOrKey") String keyOrUrl, @Name("workspace") String workspace, @Name("data") Object data,@Name(value = "weightproperty",defaultValue = "null") String weightproperty,@Name(value = "exportproperties",defaultValue = "[]") List<String> exportproperties) {
        if (workspace == null) workspace = "workspace0";
        String url = getGephiUrl(keyOrUrl)+"/"+Util.encodeUrlComponent(workspace)+"?operation=updateGraph";
        long start = System.currentTimeMillis();
        HashSet<Node> nodes = new HashSet<>(1000);
        HashSet<Relationship> rels = new HashSet<>(10000);
        List<String> propertyNames = new ArrayList<>(exportproperties);
        propertyNames.removeAll(RESERVED);
        if (Graphs.extract(data, nodes, rels)) {
            String payload = toGephiStreaming(nodes, rels, weightproperty, propertyNames.toArray(new String[propertyNames.size()]));
            JsonUtil.loadJson(url,map("method","POST","Content-Type","application/json; charset=utf-8"), payload).count();
            return Stream.of(new ProgressInfo(url,"graph","gephi").update(nodes.size(),rels.size(),nodes.size()).done(start));
        }
        return Stream.empty();
    }
{'fraction_non_alphanumeric': 0.13870696893366918, 'fraction_numerical': 0.015449202350965575, 'mean_word_length': 4.54046511627907, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6797421', 'n_tokens_mistral': 1864, 'n_tokens_neox': 1779, 'n_words': 383}