<commit_before>package org.myrobotlab.service;

import java.util.ArrayList;
import java.util.List;

import org.myrobotlab.framework.Service;
import org.myrobotlab.framework.ServiceType;
import org.myrobotlab.logging.Level;
import org.myrobotlab.logging.LoggerFactory;
import org.myrobotlab.logging.Logging;
import org.myrobotlab.logging.LoggingFactory;
import org.slf4j.Logger;
import org.wikidata.wdtk.datamodel.interfaces.EntityDocument;
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.QuantityValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
import org.wikidata.wdtk.datamodel.interfaces.Value;
import org.wikidata.wdtk.datamodel.interfaces.ValueSnak;
import org.wikidata.wdtk.datamodel.json.jackson.JacksonValueSnak;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher;
import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException;

public class WikiDataFetcher extends Service {

  private static final long serialVersionUID = 1L;

  public final static Logger log = LoggerFactory.getLogger(WikiDataFetcher.class);

  static String language = "en";
  static String website = "enwiki";

  public static void main(String[] args) {
    LoggingFactory.init(Level.INFO);

    try {
      WikiDataFetcher wdf = (WikiDataFetcher) Runtime.start("wikiDataFetcher", "WikiDataFetcher");
      wdf.setWebSite("enwiki");
      wdf.setLanguage("en");
      
      String start = wdf.grabStart("halloween").toLowerCase();
      
      String query = "halloween";
      String desc = wdf.getDescription(query);
      log.info(query + " is " + desc);

      
      query = "empire state building";
      desc = wdf.getDescription(query);
      log.info(query + " is " + desc);
     
      query = "the pyramids";
      desc = wdf.getDescription(query);
      log.info(query + " is " + desc);
     
      query = "dog";
      desc = wdf.getDescription(query);
      log.info(query + " is " + desc);
     
      
      EntityDocument doc = WikiDataFetcher.getWiki("Halloween");

      log.info(getData("united states", "P36"));

      log.info(doc.toString());

      log.info(getData("eiffel tower", "P2048"));

      log.info(getData("nothing to test", "P2048"));

    } catch (Exception e) {
      Logging.logError(e);
    }
  }

  public WikiDataFetcher(String n, String id) {
    super(n, id);
  }

  public String[] getCategories() {
    return new String[] { "general" };
  }

  @Override
  public String getDescription() {
    return "Used to collect datas from wikidata";
  }

  public void setLanguage(String lang) {
    language = lang;
  }

  public void setWebSite(String site) {
    website = site;<commit_msg>clean<commit_after>package org.myrobotlab.service;

import java.util.ArrayList;
import java.util.List;

import org.myrobotlab.framework.Service;
import org.myrobotlab.logging.Level;
import org.myrobotlab.logging.LoggerFactory;
import org.myrobotlab.logging.Logging;
import org.myrobotlab.logging.LoggingFactory;
import org.slf4j.Logger;
import org.wikidata.wdtk.datamodel.interfaces.EntityDocument;
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.QuantityValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
import org.wikidata.wdtk.datamodel.interfaces.Value;
import org.wikidata.wdtk.datamodel.interfaces.ValueSnak;
import org.wikidata.wdtk.datamodel.json.jackson.JacksonValueSnak;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher;
import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException;

public class WikiDataFetcher extends Service {

  private static final long serialVersionUID = 1L;

  public final static Logger log = LoggerFactory.getLogger(WikiDataFetcher.class);

  static String language = "en";
  static String website = "enwiki";

  public static void main(String[] args) {
    LoggingFactory.init(Level.INFO);

    try {
      WikiDataFetcher wdf = (WikiDataFetcher) Runtime.start("wikiDataFetcher", "WikiDataFetcher");
      wdf.setWebSite("enwiki");
      wdf.setLanguage("en");
      
      String start = wdf.grabStart("halloween").toLowerCase();
      
      String query = "halloween";
      String desc = wdf.getDescription(query);
      log.info(query + " is " + desc);

      
      query = "empire state building";
      desc = wdf.getDescription(query);
      log.info(query + " is " + desc);
     
      query = "the pyramids";
      desc = wdf.getDescription(query);
      log.info(query + " is " + desc);
     
      query = "dog";
      desc = wdf.getDescription(query);
      log.info(query + " is " + desc);
     
      
      EntityDocument doc = WikiDataFetcher.getWiki("Halloween");

      log.info(getData("united states", "P36"));

      log.info(doc.toString());

      log.info(getData("eiffel tower", "P2048"));

      log.info(getData("nothing to test", "P2048"));

    } catch (Exception e) {
      log.error("main threw", e);
    }
  }

  public WikiDataFetcher(String n, String id) {
    super(n, id);
  }

  public String[] getCategories() {
    return new String[] { "general" };
  }

  @Override
  public String getDescription() {
    return "Used to collect datas from wikidata";
  }

  public void setLanguage(String lang) {
    language = lang;
  }

  public void setWebSite(String site) {
    website = site;
{'fraction_non_alphanumeric': 0.11695028361695028, 'fraction_numerical': 0.004004004004004004, 'mean_word_length': 3.897875816993464, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1084677', 'n_tokens_mistral': 2066, 'n_tokens_neox': 2004, 'n_words': 355}