<commit_before># Parse JSON data returned by Special:Entity API of Wikidata
# and convert it to CSV format

# headers
[
  "Entity",
  "Group",
  "Value Name",
  "Value Type",
  "Value",
  "Rank",
  "Start Time",
  "End Time"
]
,
(
  # records
  .entities
  | .[]
  | [
      "",
      "",
      "",
      "",
      "",
      "",
      "",
      ""
    ] as $empty_record
  | .id as $entity
  | (
      .aliases
      | .[]
      | .[]
      | [
          $entity,
          "alias",
          "language",
          .language,
          .value,
          "",
          "",
          ""
        ]
    ),
    $empty_record,
    (
      .descriptions
      | .[]
      | [
          $entity,
          "description",
          "language",
          .language,
          .value,
          "",
          "",
          ""
        ]
    ),
    $empty_record,
    (
      .labels
      | .[]
      | [
          $entity,
          "label",
          "language",
          .language,
          .value,
          "",
          "",
          ""
        ]
    ),
    $empty_record,
    (
      .claims
      | .[]
      | .[]
      | .rank as $rank
      | .qualifiers.P580[0].datavalue.value.time as $start_time
      | .qualifiers.P582[0].datavalue.value.time as $end_time
      | .mainsnak
      | .property as $value_name
      | if .snaktype == "novalue"
        then
          empty
        elif .datatype == "globe-coordinate"
        then
          .datavalue.value
          | { latitude, longitude, altitude }
          | to_entries
          | .[]
          | .key as $value_type
          | .value
          | values
          | [
              $entity,
              "claim",
              $value_name,
              $value_type,
              .,
              $rank,
              $start_time,
              $end_time
            ]
        else
          .datatype as $value_type
          | (
              .datavalue.value
              | (
                     scalars
                  // ."numeric-id"
                  // .amount
                  // .text
                  // .time
                )
            )
          | [
              $entity,
              "claim",
              $value_name,
              $value_type,
              .,
              $rank,
              $start_time,
              $end_time
            ]
      end
    ),
    $empty_record,
    (
      .sitelinks
      | .[]
      | [
          $entity,
          "sitelink",
          "site",
          .site,
          .title,
          "",
          "",
          ""
        ]
    )
)
# Note: @csv requires --raw-output parameter on jq command line
| @csv<commit_msg>Add position of claims (per property) and aliases (per language)<commit_after># Parse JSON data returned by Special:Entity API of Wikidata
# and convert it to CSV format

# headers
[
  "Entity",
  "Group",
  "Value Name",
  "Value Type",
  "Value Position",
  "Value",
  "Rank",
  "Start Time",
  "End Time"
]
,
(
  # records
  .entities[]
  | [
      "",
      "",
      "",
      "",
      "",
      "",
      "",
      "",
      ""
    ] as $empty_record
  | .id as $entity
  | (
      .aliases[]
      | to_entries
      | .[]
      | (.key + 1) as $position
      | .value
      | [
          $entity,
          "alias",
          "language",
          .language,
          $position,
          .value,
          "",
          "",
          ""
        ]
    ),
    $empty_record,
    (
      .descriptions[]
      | [
          $entity,
          "description",
          "language",
          .language,
          "",
          .value,
          "",
          "",
          ""
        ]
    ),
    $empty_record,
    (
      .labels[]
      | [
          $entity,
          "label",
          "language",
          .language,
          "",
          .value,
          "",
          "",
          ""
        ]
    ),
    $empty_record,
    (
      .claims[]
      | to_entries
      | .[]
      | (.key + 1) as $position
      | .value
      | .rank as $rank
      | .qualifiers.P580[0].datavalue.value.time as $start_time
      | .qualifiers.P582[0].datavalue.value.time as $end_time
      | .mainsnak
      | .property as $value_name
      | if .snaktype == "novalue"
        then
          empty
        elif .datatype == "globe-coordinate"
        then
          .datavalue.value
          | { latitude, longitude, altitude }
          | to_entries
          | .[]
          | .key as $value_type
          | .value
          | values
          | [
              $entity,
              "claim",
              $value_name,
              $value_type,
              $position,
              .,
              $rank,
              $start_time,
              $end_time
            ]
        else
          .datatype as $value_type
          | (
              .datavalue.value
              | (
                     scalars
                  // ."numeric-id"
                  // .amount
                  // .text
                  // .time
                )
            )
          | [
              $entity,
              "claim",
              $value_name,
              $value_type,
              $position,
              .,
              $rank,
              $start_time,
              $end_time
            ]
      end
    ),
    $empty_record,
    (
      .sitelinks[]
      | [
          $entity,
          "sitelink",
          "site",
          .site,
          "",
          .title,
          "",
          "",
          ""
        ]
    )
)
# Note: @csv requires --raw-output parameter on jq command line
| @csv
{'fraction_non_alphanumeric': 0.12952346829093514, 'fraction_numerical': 0.0032246506628448583, 'mean_word_length': 0.8263002944062807, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5664752', 'n_tokens_mistral': 1658, 'n_tokens_neox': 1656, 'n_words': 289}