<commit_before>{
  "title": "Book metadata update message.",
  "type": "object",
  "properties": {
    "classification": { "$ref": "#/definitions/classification" },
    "data": {
      "anyOf": [
        { "$ref": "#/definitions/book_data" },
        { "$ref": "#/definitions/media_data" }
      ]
    },
    "source": {
      "type": "object",
      "properties": {
        "delivered_at": {
          "title": "The ISO 8601 timestamp of when this asset is considered to be valid from.",
          "type": "string",
          "format": "date-time"
        },
        "uri": {
          "title": "The location of the source asset for the metadata described.",
          "type": "string",
          "format": "uri"
        },
        "role": {
          "title": "The role of the source that provided this data.",
          "enum": ["publisher ftp", "content manager"]
        },
        "username": {
          "title": "The username of the source entity",
          "type": "string"
        },
        "system": {
          "title": "The system which processed this asset. Not required as the message metadata contains the fallback.",
          "type": "string"
        },
        "processed_at": {
          "title": "The ISO 8601 timestamp of when this asset was processed. Not required as the timestamp of the message is the fallback.",
          "type": "string",
          "format": "date-time"
        }
      },
      "required": ["delivered_at", "uri", "role", "username"],
      "additionalProperties": false
    }
  },
  "required": ["classification", "data", "source"],
  "additionalProperties": false,
  "definitions": {
    "iso639-2b": {
      "title": "The list of ISO-639/2B language codes, for validating languages recognised by blinkbox books.",
      "enum": ["aar","abk","ace","ach","ada","ady","afa","afh","afr","ain","aka","akk","alb","ale","alg","alt","amh","ang","anp","apa","ara","arc","arg","arm","arn","arp","art","arw","asm","ast","ath","aus","ava","ave","awa","aym","aze","bad","bai","bak","bal","bam","ban","baq","bas","bat","bej","bel","bem","ben","ber","bho","bih","bik","bin","bis","bla","bnt","bos","bra","bre","btk","bua","bug","bul","bur","byn","cad","cai","car","cat","cau","ceb","cel","cha","chb","che","chg","chi","chk","chm","chn","cho","chp","chr","chu","chv","chy","cmc","cop","cor","cos","cpe","cpf","cpp","cre","crh","crp","csb","cus","cze","dak","dan","dar","day","del","den","dgr","din","div","doi","dra","dsb","dua","dum","dut","dyu","dzo","efi","egy","eka","elx","eng","enm","epo","est","ewe","ewo","fan","fao","fat","fij","fil","fin","fiu","fon","fre","frm","fro","frr","frs","fry","ful","fur","gaa","gay","gba","gem","geo","ger","gez","gil","gla","gle","glg","glv","gmh","goh","gon","gor","got","grb","grc","gre","grn","gsw","guj","gwi","hai","hat","hau","haw","heb","her","hil","him","hin","hit","hmn","hmo","hrv","hsb","hun","hup","iba","ibo","ice","ido","iii","ijo","iku","ile","ilo","ina","inc","ind","ine","inh","ipk","ira","iro","ita","jav","jbo","jpn","jpr","jrb","kaa","kab","kac","kal","kam","kan","kar","kas","kau","kaw","kaz","kbd","kha","khi","khm","kho","kik","kin","kir","kmb","kok","kom","kon","kor","kos","kpe","krc","krl","kro","kru","kua","kum","kur","kut","lad","lah","lam","lao","lat","lav","lez","lim","lin","lit","lol","loz","ltz","lua","lub","lug","lui","lun","luo","lus","mac","mad","mag","mah","mai","mak","mal","man","mao","map","mar","mas","may","mdf","mdr","men","mga","mic","min","mis","mkh","mlg","mlt","mnc","mni","mno","moh","mol","mon","mos","mul","mun","mus","mwl","mwr","myn","myv","nah","nai","nap","nau","nav","nbl","nde","ndo","nds","nep","new","nia","nic","niu","nno","nob","nog","non","nor","nqo","nso","nub","nwc","nya","nym","nyn","nyo","nzi","oci","oji","ori","orm","osa","oss","ota","oto","paa","pag","pal","pam","pan","pap","pau","peo","per","phi","phn","pli","pol","pon","por","pra","pro","pus","qar","qav","que","raj","rap","rar","roa","roh","rom","rum","run","rup","rus","sad","sag","sah","sai","sal","sam","san","sas","sat","scc","scn","sco","scr","sel","sem","sga","sgn","shn","sid","sin","sio","sit","sla","slo","slv","sma","sme","smi","smj","smn","smo","sms","sna","snd","snk","sog","som","son","sot","spa","srd","srn","srp","srr","ssa","ssw","suk","sun","sus","sux","swa","swe","syc","syr","tah","tai","tam","tat","tel","tem","ter","tet","tgk","tgl","tha","tib","tig","tir","tiv","tkl","tlh","tli","tmh","tog","ton","tpi","tsi","tsn","tso","tuk","tum","tup","tur","tut","tvl","twi","tyv","udm","uga","uig","ukr","umb","und","urd","uzb","vai","ven","vie","vol","vot","wak","wal","war","was","wel","wen","wln","wol","xal","xho","yao","yap","yid","yor","ypk","zap","zbl","zen","zha","znd","zul","zun","zxx","zza"]
    },
    "classification": {
      "title": "An ordered list of identifiers classifying this entity, most general at index 0.",
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "realm": {
            "title": "The name of the realm in which this unique identifier exists",
            "enum": ["isbn", "contributor_id", "source_role", "type"]
          },
          "id": {
            "type": "string"
          }
        },
        "required": ["realm", "id"],
        "additionalProperties": false
      }
    },
    "book_data": {
      "title": "Metadata about a book, as would be collected from ONIX.",
      "properties": {
        "title": {
          "title": "The title of the book",
          "type": "string"
        },
        "subtitle": {
          "title": "The subtitle of the book",
          "type": "string"
        },
        "contributors": {
          "title": "The array listing, in order, contributors this work is attributed to.",
          "type": "array",
          "items": {
            "title": "The details of the contributor",
            "type": "object",
            "properties": {
              "classification":  { "$ref": "#/definitions/classification" },
              "names": {
                "type": "object",
                "properties": {
                  "display": {
                    "title": "The name of the contributor as it should appear in general text.",
                    "type": "string"
                  },
                  "sort": {
                    "title": "The string that should be used for sorting this contributor among others.",
                    "type": "string"
                  }
                  // TODO: Add name components
                },
                "required": ["display"],
                "additionalProperties": false
              },
              "role": {
                "type": "string"
              }
            },
            "required": ["classification", "names", "role"]
          }
        },
        // TODO: Availability
        "availability": {},
        "dates": {
          "title": "Information about dates relevant to the publishing of this book.",
          "type": "object",
          "properties": {
            "publish": {
              "title": "The date and time at which the publisher allows this book to go on sale.",
              "type": "string",
              "format": "date-time"
            },
            "announce": {
              "title": "The date and time at which the publisher allows this book to be announced - ie. have metadata released.",
              "type": "string",
              "format": "date-time"
            }
          },
          "additionalProperties": false
        },
        // TODO: Descriptions
        "description": {},
        "language": {
          "title": "The languages (ISO-639/2B) the book is written in. Order is significant, with the 0 indexed language being the most promenant.",
          "type": "array",
          "items": {
            "$ref": "#/definitions/iso639-2b"
          }
        },
        // TODO: Sellable in / Not sellable in
        "pages": {
          "title": "Number of pages in an equivalent physical book",
          "type": "integer"
        },
        "publisher": {
          "title": "The name of the publisher as it should appear on the site.",
          "type": "string"
        },
        "imprint": {
          "title": "The name of the publisher's imprint as it should appear on the site.",
          "type": "string"
        },
        // TODO: Prices
        "prices": {},
        // TODO: Subjects
        "subjects": {}
      },
      "additionalProperties": false
    },
    "media_data": {
      "title": "Holds information about media for this book.",
      "type": "object",
      "properties": {
        "media": {
          "anyOf": [
            { "$ref": "#/definitions/epub_media" },
            { "$ref": "#/definitions/image_media" }
          ]
        }
      },
      "additionalProperties": false
    },
    "epub_media": {
      "title": "This holds info about one or more epubs.",
      "type": "object",
      "properties": {
        "epubs": {
          "type": "array",
          "items": {
            "allOf": [
              { "$ref": "#/definitions/media_item" },
              {
                "title": "The additional components of the media item which are specific to epubs.",
                "properties": {
                  "keyfile": {
                    "title": "The location of the (secured) keyfile for decrypting the ePub. The absence of this key implies it is not encrypted.",
                    "type": "string",
                    "format": "uri"
                  },
                  "wordcount": {
                    "title": "The number of words in the associated epub.",
                    "type": "integer"
                  }
                },
                "required": ["wordcount"]
              }
            ]
          }
        }
      },
      "additionalProperties": false
    },
    "image_media": {
      "title": "This holds info about one or more images and can be applied to a book or a contributor.",
      "type": "object",
      "properties": {
        "images": {
          "type": "array",
          "items": {
            "allOf": [
              { "$ref": "#/definitions/media_item" },
              {
                "title": "The additional components of the media item which are specific to images.",
                "properties": {
                  "width": {
                    "title": "The width of the image in pixels.",
                    "type": "integer"
                  },
                  "height": {
                    "title": "The height of the image in pixels.",
                    "type": "integer"
                  }
                }
              }
            ]
          }
        }
      },
      "additionalProperties": false
    },
    "media_item": {
      "title": "All images, epubs, mobis etc are described using this format.",
      "type": "object",
      "properties": {
        "classification":  { "$ref": "#/definitions/classification" },
        "uris": {
          "type": "array",
          "items": {
            "properties": {
              "type": {
                "enum": ["static", "resource_server"]
              },
              "uri": {
                "type": "string",
                "format": "uri"
              },
              "params": {
                "type": "string",
                "format": "resource server params"
              }
            },
            "required": ["type", "uri"],
            "additionalProperties": false
          }
        },
        "size": {
          "title": "The size of the asset in bytes",
          "type": "integer"
        }
      },
      "required": ["classification", "uris", "size"]
    }
  }
}<commit_msg>Subjects & series<commit_after>{
  "title": "Book metadata update message.",
  "type": "object",
  "properties": {
    "classification": { "$ref": "#/definitions/classification" },
    "data": {
      "anyOf": [
        { "$ref": "#/definitions/book_data" },
        { "$ref": "#/definitions/media_data" }
      ]
    },
    "source": {
      "type": "object",
      "properties": {
        "delivered_at": {
          "title": "The ISO 8601 timestamp of when this asset is considered to be valid from.",
          "type": "string",
          "format": "date-time"
        },
        "uri": {
          "title": "The location of the source asset for the metadata described.",
          "type": "string",
          "format": "uri"
        },
        "role": {
          "title": "The role of the source that provided this data.",
          "enum": ["publisher ftp", "content manager"]
        },
        "username": {
          "title": "The username of the source entity",
          "type": "string"
        },
        "system": {
          "title": "The system which processed this asset. Not required as the message metadata contains the fallback.",
          "type": "string"
        },
        "processed_at": {
          "title": "The ISO 8601 timestamp of when this asset was processed. Not required as the timestamp of the message is the fallback.",
          "type": "string",
          "format": "date-time"
        }
      },
      "required": ["delivered_at", "uri", "role", "username"],
      "additionalProperties": false
    }
  },
  "required": ["classification", "data", "source"],
  "additionalProperties": false,
  "definitions": {
    "iso639-2b": {
      "title": "The list of ISO-639/2B language codes, for validating languages recognised by blinkbox books.",
      "enum": ["aar","abk","ace","ach","ada","ady","afa","afh","afr","ain","aka","akk","alb","ale","alg","alt","amh","ang","anp","apa","ara","arc","arg","arm","arn","arp","art","arw","asm","ast","ath","aus","ava","ave","awa","aym","aze","bad","bai","bak","bal","bam","ban","baq","bas","bat","bej","bel","bem","ben","ber","bho","bih","bik","bin","bis","bla","bnt","bos","bra","bre","btk","bua","bug","bul","bur","byn","cad","cai","car","cat","cau","ceb","cel","cha","chb","che","chg","chi","chk","chm","chn","cho","chp","chr","chu","chv","chy","cmc","cop","cor","cos","cpe","cpf","cpp","cre","crh","crp","csb","cus","cze","dak","dan","dar","day","del","den","dgr","din","div","doi","dra","dsb","dua","dum","dut","dyu","dzo","efi","egy","eka","elx","eng","enm","epo","est","ewe","ewo","fan","fao","fat","fij","fil","fin","fiu","fon","fre","frm","fro","frr","frs","fry","ful","fur","gaa","gay","gba","gem","geo","ger","gez","gil","gla","gle","glg","glv","gmh","goh","gon","gor","got","grb","grc","gre","grn","gsw","guj","gwi","hai","hat","hau","haw","heb","her","hil","him","hin","hit","hmn","hmo","hrv","hsb","hun","hup","iba","ibo","ice","ido","iii","ijo","iku","ile","ilo","ina","inc","ind","ine","inh","ipk","ira","iro","ita","jav","jbo","jpn","jpr","jrb","kaa","kab","kac","kal","kam","kan","kar","kas","kau","kaw","kaz","kbd","kha","khi","khm","kho","kik","kin","kir","kmb","kok","kom","kon","kor","kos","kpe","krc","krl","kro","kru","kua","kum","kur","kut","lad","lah","lam","lao","lat","lav","lez","lim","lin","lit","lol","loz","ltz","lua","lub","lug","lui","lun","luo","lus","mac","mad","mag","mah","mai","mak","mal","man","mao","map","mar","mas","may","mdf","mdr","men","mga","mic","min","mis","mkh","mlg","mlt","mnc","mni","mno","moh","mol","mon","mos","mul","mun","mus","mwl","mwr","myn","myv","nah","nai","nap","nau","nav","nbl","nde","ndo","nds","nep","new","nia","nic","niu","nno","nob","nog","non","nor","nqo","nso","nub","nwc","nya","nym","nyn","nyo","nzi","oci","oji","ori","orm","osa","oss","ota","oto","paa","pag","pal","pam","pan","pap","pau","peo","per","phi","phn","pli","pol","pon","por","pra","pro","pus","qar","qav","que","raj","rap","rar","roa","roh","rom","rum","run","rup","rus","sad","sag","sah","sai","sal","sam","san","sas","sat","scc","scn","sco","scr","sel","sem","sga","sgn","shn","sid","sin","sio","sit","sla","slo","slv","sma","sme","smi","smj","smn","smo","sms","sna","snd","snk","sog","som","son","sot","spa","srd","srn","srp","srr","ssa","ssw","suk","sun","sus","sux","swa","swe","syc","syr","tah","tai","tam","tat","tel","tem","ter","tet","tgk","tgl","tha","tib","tig","tir","tiv","tkl","tlh","tli","tmh","tog","ton","tpi","tsi","tsn","tso","tuk","tum","tup","tur","tut","tvl","twi","tyv","udm","uga","uig","ukr","umb","und","urd","uzb","vai","ven","vie","vol","vot","wak","wal","war","was","wel","wen","wln","wol","xal","xho","yao","yap","yid","yor","ypk","zap","zbl","zen","zha","znd","zul","zun","zxx","zza"]
    },
    "classification": {
      "title": "An ordered list of identifiers classifying this entity, most general at index 0.",
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "realm": {
            "title": "The name of the realm in which this unique identifier exists",
            "enum": ["isbn", "contributor_id", "source_role", "type"]
          },
          "id": {
            "type": "string"
          }
        },
        "required": ["realm", "id"],
        "additionalProperties": false
      }
    },
    "book_data": {
      "title": "Metadata about a book, as would be collected from ONIX.",
      "properties": {
        "title": {
          "title": "The title of the book",
          "type": "string"
        },
        "subtitle": {
          "title": "The subtitle of the book",
          "type": "string"
        },
        "contributors": {
          "title": "The array listing, in order, contributors this work is attributed to.",
          "type": "array",
          "items": {
            "title": "The details of the contributor",
            "type": "object",
            "properties": {
              "classification":  { "$ref": "#/definitions/classification" },
              "names": {
                "type": "object",
                "properties": {
                  "display": {
                    "title": "The name of the contributor as it should appear in general text.",
                    "type": "string"
                  },
                  "sort": {
                    "title": "The string that should be used for sorting this contributor among others.",
                    "type": "string"
                  }
                  // TODO: Add name components
                },
                "required": ["display"],
                "additionalProperties": false
              },
              "role": {
                "type": "string"
              }
            },
            "required": ["classification", "names", "role"]
          }
        },
        // TODO: Availability
        "availability": {},
        "dates": {
          "title": "Information about dates relevant to the publishing of this book.",
          "type": "object",
          "properties": {
            "publish": {
              "title": "The date and time at which the publisher allows this book to go on sale.",
              "type": "string",
              "format": "date-time"
            },
            "announce": {
              "title": "The date and time at which the publisher allows this book to be announced - ie. have metadata released.",
              "type": "string",
              "format": "date-time"
            }
          },
          "additionalProperties": false
        },
        // TODO: Descriptions
        "description": {},
        "language": {
          "title": "The languages (ISO-639/2B) the book is written in. Order is significant, with the 0 indexed language being the most promenant.",
          "type": "array",
          "items": {
            "$ref": "#/definitions/iso639-2b"
          }
        },
        // TODO: Sellable in / Not sellable in
        "pages": {
          "title": "Number of pages in an equivalent physical book",
          "type": "integer"
        },
        "publisher": {
          "title": "The name of the publisher as it should appear on the site.",
          "type": "string"
        },
        "imprint": {
          "title": "The name of the publisher's imprint as it should appear on the site.",
          "type": "string"
        },
        // TODO: Prices
        "prices": {},
        "subjects": {
          "title": "Categorisation of the book; keywords, genres etc.",
          "type": "array",
          "items": {
            "oneOf": [
              {
                "title": "BISAC codes",
                "type": "object",
                "properties": {
                  "type": {
                    "enum": ["BISAC"]
                  },
                  "code": {
                    "title": "The BISAC code",
                    // TODO: enum of valid BISAC codes
                    "type": "string"
                  }
                },
                "required": ["type", "code"],
                "additionalProperties": false
              },
              {
                "title": "BIC codes must be one of the accepted values",
                "type": "object",
                "properties": {
                  "type": {
                    "enum": ["BIC"]
                  },
                  "code": {
                    "title": "The BIC code",
                    // TODO: enum of valid BIC codes
                    "type": "string"
                  }
                },
                "required": ["type", "code"],
                "additionalProperties": false
              },
              {
                "title": "Anything that isn't BIC or BISAC codes we just take as informative",
                "type": "object",
                "properties": {
                  "type": {
                    "type": "string"
                    // TODO: Disallowed values: BIC and BISAC
                  },
                  "code": {
                    "title": "The text used for this subject code",
                    "type": "string"
                  }
                },
                "required": ["type", "code"],
                "additionalProperties": false
              }
            ]
          }
        },
        "series": {
          "title": "Details about the series this book is in.",
          "type": "object",
          "properties": {
            "title": {
              "title": "The title of the series this book is in. This should be globally unique, but I have no clue how we can ensure that...",
              "type": "string"
            },
            "number": {
              "title": "This book's number in the series",
              "type": "integer",
              "minimum": 1
            }
          },
          "required": ["title"],
          "additionalProperties": false
        }
      },
      "additionalProperties": false
    },
    "media_data": {
      "title": "Holds information about media for this book.",
      "type": "object",
      "properties": {
        "media": {
          "anyOf": [
            { "$ref": "#/definitions/epub_media" },
            { "$ref": "#/definitions/image_media" }
          ]
        }
      },
      "additionalProperties": false
    },
    "epub_media": {
      "title": "This holds info about one or more epubs.",
      "type": "object",
      "properties": {
        "epubs": {
          "type": "array",
          "items": {
            "allOf": [
              { "$ref": "#/definitions/media_item" },
              {
                "title": "The additional components of the media item which are specific to epubs.",
                "properties": {
                  "keyfile": {
                    "title": "The location of the (secured) keyfile for decrypting the ePub. The absence of this key implies it is not encrypted.",
                    "type": "string",
                    "format": "uri"
                  },
                  "wordcount": {
                    "title": "The number of words in the associated epub.",
                    "type": "integer"
                  }
                },
                "required": ["wordcount"]
              }
            ]
          }
        }
      },
      "additionalProperties": false
    },
    "image_media": {
      "title": "This holds info about one or more images and can be applied to a book or a contributor.",
      "type": "object",
      "properties": {
        "images": {
          "type": "array",
          "items": {
            "allOf": [
              { "$ref": "#/definitions/media_item" },
              {
                "title": "The additional components of the media item which are specific to images.",
                "properties": {
                  "width": {
                    "title": "The width of the image in pixels.",
                    "type": "integer"
                  },
                  "height": {
                    "title": "The height of the image in pixels.",
                    "type": "integer"
                  }
                }
              }
            ]
          }
        }
      },
      "additionalProperties": false
    },
    "media_item": {
      "title": "All images, epubs, mobis etc are described using this format.",
      "type": "object",
      "properties": {
        "classification":  { "$ref": "#/definitions/classification" },
        "uris": {
          "type": "array",
          "items": {
            "properties": {
              "type": {
                "enum": ["static", "resource_server"]
              },
              "uri": {
                "type": "string",
                "format": "uri"
              },
              "params": {
                "type": "string",
                "format": "resource server params"
              }
            },
            "required": ["type", "uri"],
            "additionalProperties": false
          }
        },
        "size": {
          "title": "The size of the asset in bytes",
          "type": "integer"
        }
      },
      "required": ["classification", "uris", "size"]
    }
  }
}
{'fraction_non_alphanumeric': 0.22214471185521065, 'fraction_numerical': 0.0020540247258070765, 'mean_word_length': 2.013429872708163, 'pattern_counts': {'":': 425, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5380596', 'n_tokens_mistral': 7741, 'n_tokens_neox': 7433, 'n_words': 1560}