<jupyter_start><jupyter_text>Custom Spacy Rules<jupyter_code>doc = {"doc_id": "de60d1fd856b8c49568eba4f5026c58c8e7c09f4", "measurement": {"date": "2013-01-28", "type": "Measurement", "number": 113.92, "timeseries": "77e7e461cf45417a99c3d0f5eb17ab3533093495"}}
etk_config = {
    "document_id": "doc_id",
    "resources": {
    "stop_word_dictionaries": {},
    "landmark": [],
    "dictionaries": {},
    "spacy_field_rules": {
      "measurement_date_rules": "/Users/amandeep/Github/etk/etk/resources/measurement_date_rules.json"
    }
  },
  "content_extraction": {
    "json_content": [
      {
        "input_path": "measurement.type", 
        "segment_name": "measurement__type"
      }, 
      {
        "input_path": "measurement.date", 
        "segment_name": "measurement__date"
      }, 
      {
        "input_path": "measurement.number", 
        "segment_name": "measurement__number"
      }, 
      {
        "input_path": "measurement.timeseries", 
        "segment_name": "measurement__timeseries"
      }
    ]
  }, 
  "data_extraction": [
    {
      "fields": {
        "type": {
          "extractors": {
            "extract_as_is": {}
          }
        }
      }, 
      "input_path": "content_extraction.measurement__type[*]"
    }, 
    {
      "fields": {
        "event_date": {
          "extractors": {
            "extract_as_is": {
              "config": {
                "ignore_past_years": 100, 
                "post_filter": "parse_date_generic"
              }
            },
            "extract_using_custom_spacy": {
                "config": {
                  "spacy_field_rules": "measurement_date_rules",
                  "post_filter": "parse_date_generic"
                }
              }
          }
        }
      }, 
      "input_path": "content_extraction.measurement__date[*]"
    }, 
    {
      "fields": {
        "value": {
          "extractors": {
            "extract_as_is": {}
          }
        }
      }, 
      "input_path": "content_extraction.measurement__number[*]"
    }, 
    {
      "fields": {
        "measure": {
          "extractors": {
            "extract_as_is": {}
          }
        }
      }, 
      "input_path": "content_extraction.measurement__timeseries[*]"
    }
  ]
}
import json
from etk.core import Core

c = Core(extraction_config=etk_config)
r = c.process(doc)
print json.dumps(r["knowledge_graph"]["event_date"], indent=2)<jupyter_output>[
  {
    "confidence": 1.0, 
    "provenance": [
      {
        "source": {
          "segment": "content_extraction.measurement__date.[0]", 
          "document_id": "de60d1fd856b8c49568eba4f5026c58c8e7c09f4"
        }, 
        "confidence": {
          "extraction": 1.0
        }, 
        "method": "extract_as_is", 
        "extracted_value": "2013-01-28T00:00:00"
      }, 
      {
        "source": {
          "segment": "other_segment", 
          "context": {
            "end": 5, 
            "text": " <etk 'attribute' = 'event_date'>2013 - 01 - 28</etk>  ", 
            "start": 0, 
            "input": "tokens", 
            "identifier": "rule_4", 
            "rule_id": 3
          }, 
          "document_id": "de60d1fd856b8c49568eba4f5026c58c8e7c09f4"
        }, 
        "confidence": {
          "extraction": 1.0
        }, 
        "method": "extract_using_custom_spacy", 
        "extracted_value": "2013-01-28T00:00:00"
      }
    ], 
    "key": "2013-01-28t00:00:00",[...]
{'fraction_non_alphanumeric': 0.15355587808417998, 'fraction_numerical': 0.05166908563134978, 'mean_word_length': 1.8269073010664478, 'pattern_counts': {'":': 75, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '469279', 'license': 'MIT', 'n_tokens_mistral': 1311, 'n_tokens_neox': 1160, 'n_words': 137, 'path': 'etk_date_test.ipynb', 'repo_name': 'iAmeeth/etk', 'size': '3445'}