<commit_before>// Copyright 2019 Global Phasing Ltd.
//
// Function read_metadata_from_remarks() that interprets REMARK 3
// and REMARK 200/230/240 filling in Metadata.

#ifndef GEMMI_REMARKS_HPP_
#define GEMMI_REMARKS_HPP_

#include "metadata.hpp"
#include "pdb.hpp"  // for pdb_date_format_to_iso, read_int, ...

namespace gemmi {

namespace pdb_impl {

inline bool is_double(const char* p) {
  while (std::isspace(*p)) ++p;
  if (*p == '-' || *p == '+') ++p;
  while (is_digit(*p)) ++p;
  if (*p == '.') {
    ++p;
    while (is_digit(*++p)) ++p;
  }
  while (std::isspace(*p)) ++p;
  return *p == '\0';
}

inline bool is_tls_item(const std::string& key) {
  return key.size() == 3 &&
    (key[0] == 'T' || key[0] == 'L' || key[0] == 'S') &&
    (key[1] == '1' || key[1] == '2' || key[1] == '3') &&
    (key[2] == '1' || key[2] == '2' || key[2] == '3');
}

// Usually we have one program per line:
//   XDS
//   XDS VERSION NOVEMBER 3, 2014
//   AIMLESS 0.5.17
// but it can also be a list of programs:
//   autoPROC (Version 1.3.0), AIMLESS, STARANISO
//   autoPROC, XDS (VERSION Jan 26, 2018)
// We assume that:
// - the name has only one word (apologies to Queen of Spades,
//   Force Field X, APEX 2 and Insight II).
// - comma not followed by a digit separates programs
// - brackets and the word VERSION are to be removed from version
inline void add_software(Metadata& meta, SoftwareItem::Classification type,
                         const std::string& name) {
  for (size_t start = 0, end = 0; end != std::string::npos; start = end + 1) {
    end = name.find(',', start);
    while (end != std::string::npos &&
           name[end+1] == ' ' && is_digit(name[end+2]))
      end = name.find(',', end + 1);
    meta.software.emplace_back();
    SoftwareItem& item = meta.software.back();
    item.name = trim_str(name.substr(start, end - start));
    size_t sep = item.name.find(' ');
    if (sep != std::string::npos) {
      size_t ver_start = item.name.find_first_not_of(" (", sep + 1);
      item.version = item.name.substr(ver_start);
      item.name.resize(sep);
      if (!item.version.empty() && item.version.back() == ')')
        item.version.pop_back();
      if (istarts_with(item.version, "version "))
        item.version.erase(0, 8);
    }
    item.classification = type;
    item.pdbx_ordinal = (int) meta.software.size();
  }
}

// REMARK   3   TERM                          COUNT    WEIGHT   FUNCTION.
// REMARK   3    BOND LENGTHS              : 5760   ; 2.000  ; HARMONIC
inline void add_restraint_count_weight(RefinementInfo& ref_info,
                                       const char* key, const char* value) {
  ref_info.restr_stats.emplace_back(key);
  RefinementInfo::Restr& restr = ref_info.restr_stats.back();
  const char* endptr;
  restr.count = no_sign_atoi(value, &endptr);
  if (const char* sep = std::strchr(endptr, ';'))
    restr.weight = simple_atof(sep + 1, &endptr);
  if (const char* sep = std::strchr(endptr, ';'))
    restr.function = read_string(sep+1, 50);
}

inline void read_remark3_line(const char* line, Metadata& meta) {
  // Based on:
  // www.wwpdb.org/documentation/file-format-content/format23/remark3.html
  // and analysis of PDB files.
  // In special cases, such as joint X-ray and neutron refinement 5MOO,
  // PDB file can have two REMARK 3 blocks.
  // Generally, after "REMARK   3" we have either a header-like sentance
  // or a key:value pair with a colon, or a continuation of text from the
  // previous line.
  const char* key_start = skip_blank(line + 10);
  const char* colon = std::strchr(key_start, ':');
  const char* key_end = rtrim_cstr(key_start, colon);
  std::string key(key_start, key_end);
  if (colon) {
    const char* value = skip_blank(colon + 1);
    const char* end = rtrim_cstr(value);
    if (end - value == 4 && std::strncmp(value, "NULL", 4) == 0)
      return;
    if (same_str(key, "PROGRAM"))
      add_software(meta, SoftwareItem::Refinement, std::string(value, end));
    if (meta.refinement.empty())
      return;
    RefinementInfo& ref_info = meta.refinement.back();
    if (same_str(key, "RESOLUTION RANGE HIGH (ANGSTROMS)")) {
      ref_info.resolution_high = simple_atof(value);
    } else if (same_str(key, "RESOLUTION RANGE LOW  (ANGSTROMS)")) {
      ref_info.resolution_low = simple_atof(value);
    } else if (same_str(key, "COMPLETENESS FOR RANGE        (%)")) {
      ref_info.completeness = simple_atof(value);
    } else if (same_str(key, "NUMBER OF REFLECTIONS")) {
      ref_info.reflection_count = std::atoi(value);
    } else if (same_str(key, "CROSS-VALIDATION METHOD")) {
      ref_info.cross_validation_method = std::string(value, end);
    } else if (same_str(key, "FREE R VALUE TEST SET SELECTION")) {
      ref_info.rfree_selection_method = std::string(value, end);
    } else if (same_str(key, "R VALUE     (WORKING + TEST SET)")) {
      ref_info.r_all = simple_atof(value);
    } else if (same_str(key, "R VALUE            (WORKING SET)")) {
      ref_info.r_work = simple_atof(value);
    } else if (same_str(key, "FREE R VALUE")) {
      ref_info.r_free = simple_atof(value);
    } else if (same_str(key, "FREE R VALUE TEST SET COUNT")) {
      ref_info.rfree_set_count = atoi(value);
    } else if (same_str(key, "TOTAL NUMBER OF BINS USED")) {
      ref_info.bin_count = std::atoi(value);
    } else if (same_str(key, "BIN RESOLUTION RANGE HIGH       (A)")) {
      if (!ref_info.bins.empty())
        ref_info.bins.back().resolution_high = simple_atof(value);
    } else if (same_str(key, "BIN RESOLUTION RANGE LOW        (A)")) {
      if (!ref_info.bins.empty())
        ref_info.bins.back().resolution_low = simple_atof(value);
    } else if (same_str(key, "BIN COMPLETENESS (WORKING+TEST) (%)")) {
      if (!ref_info.bins.empty())
        ref_info.bins.back().completeness = simple_atof(value);
    } else if (same_str(key, "REFLECTIONS IN BIN   (WORKING+TEST)")) {
      if (!ref_info.bins.empty())
        ref_info.bins.back().reflection_count = std::atoi(value);
    } else if (same_str(key, "BIN R VALUE          (WORKING+TEST)")) {
      if (!ref_info.bins.empty())
        ref_info.bins.back().r_all = simple_atof(value);
    } else if (same_str(key, "BIN R VALUE           (WORKING SET)")) {
      if (!ref_info.bins.empty())
        ref_info.bins.back().r_work = simple_atof(value);
    } else if (same_str(key, "BIN FREE R VALUE")) {
      if (!ref_info.bins.empty())
        ref_info.bins.back().r_free = simple_atof(value);
    } else if (same_str(key, "BIN FREE R VALUE TEST SET COUNT")) {
      if (!ref_info.bins.empty())
        ref_info.bins.back().rfree_set_count = std::atoi(value);
    } else if (same_str(key, "FROM WILSON PLOT           (A**2)")) {
      // TODO
      // exper.b_wilson = simple_atof(value);
    } else if (same_str(key, "MEAN B VALUE      (OVERALL, A**2)")) {
      ref_info.mean_b = simple_atof(value);
    } else if (same_str(key, "B11 (A**2)")) {
      ref_info.aniso_b[0][0] = simple_atof(value);
    } else if (same_str(key, "B22 (A**2)")) {
      ref_info.aniso_b[1][1] = simple_atof(value);
    } else if (same_str(key, "B33 (A**2)")) {
      ref_info.aniso_b[2][2] = simple_atof(value);
    } else if (same_str(key, "B12 (A**2)")) {
      ref_info.aniso_b[0][1] = simple_atof(value);
    } else if (same_str(key, "B13 (A**2)")) {
      ref_info.aniso_b[0][2] = simple_atof(value);
    } else if (same_str(key, "B23 (A**2)")) {
      ref_info.aniso_b[1][2] = simple_atof(value);
    } else if (same_str(key, "ESD FROM LUZZATI PLOT                    (A)")) {
      ref_info.luzzati_error = simple_atof(value);
    } else if (same_str(key, "DPI (BLOW EQ-10) BASED ON R VALUE        (A)")) {
      ref_info.dpi_blow_r = simple_atof(value);
    } else if (same_str(key, "DPI (BLOW EQ-9) BASED ON FREE R VALUE    (A)")) {
      ref_info.dpi_blow_rfree = simple_atof(value);
    } else if (same_str(key, "DPI (CRUICKSHANK) BASED ON R VALUE       (A)")) {
      ref_info.dpi_cruickshank_r = simple_atof(value);
    } else if (same_str(key, "DPI (CRUICKSHANK) BASED ON FREE R VALUE  (A)")) {
      ref_info.dpi_cruickshank_rfree = simple_atof(value);
    } else if (same_str(key, "CORRELATION COEFFICIENT FO-FC")) {
      ref_info.cc_fo_fc = simple_atof(value);
    } else if (same_str(key, "CORRELATION COEFFICIENT FO-FC FREE")) {
      ref_info.cc_fo_fc_free = simple_atof(value);
    } else if (same_str(key, "BOND LENGTHS")) {
      add_restraint_count_weight(ref_info, "t_bond_d", value);
    } else if (same_str(key, "BOND ANGLES")) {
      add_restraint_count_weight(ref_info, "t_angle_deg", value);
    } else if (same_str(key, "TORSION ANGLES")) {
      add_restraint_count_weight(ref_info, "t_dihedral_angle_d", value);
    } else if (same_str(key, "GENERAL PLANES")) {
      add_restraint_count_weight(ref_info, "t_gen_planes", value);
    } else if (same_str(key, "ISOTROPIC THERMAL FACTORS")) {
      add_restraint_count_weight(ref_info, "t_it", value);
    } else if (same_str(key, "BAD NON-BONDED CONTACTS")) {
      add_restraint_count_weight(ref_info, "t_nbd", value);
    } else if (same_str(key, "CHIRAL IMPROPER TORSION")) {
      add_restraint_count_weight(ref_info, "t_chiral_improper_torsion", value);
    } else if (same_str(key, "IDEAL-DIST CONTACT TERM")) {
      add_restraint_count_weight(ref_info, "t_ideal_dist_contact", value);
    } else if (same_str(key, "BOND LENGTHS                       (A)")) {
      impl::find_or_add(ref_info.restr_stats, "t_bond_d").dev_ideal
        = read_double(value, 50);
    } else if (same_str(key, "BOND ANGLES                  (DEGREES)")) {
      impl::find_or_add(ref_info.restr_stats, "t_angle_deg").dev_ideal
        = read_double(value, 50);<commit_msg>reading REMARKs: handle all geometric function terms (restraint summary)<commit_after>// Copyright 2019 Global Phasing Ltd.
//
// Function read_metadata_from_remarks() that interprets REMARK 3
// and REMARK 200/230/240 filling in Metadata.

#ifndef GEMMI_REMARKS_HPP_
#define GEMMI_REMARKS_HPP_

#include "metadata.hpp"
#include "pdb.hpp"  // for pdb_date_format_to_iso, read_int, ...

namespace gemmi {

namespace pdb_impl {

inline bool is_double(const char* p) {
  while (std::isspace(*p)) ++p;
  if (*p == '-' || *p == '+') ++p;
  while (is_digit(*p)) ++p;
  if (*p == '.') {
    ++p;
    while (is_digit(*++p)) ++p;
  }
  while (std::isspace(*p)) ++p;
  return *p == '\0';
}

inline bool is_tls_item(const std::string& key) {
  return key.size() == 3 &&
    (key[0] == 'T' || key[0] == 'L' || key[0] == 'S') &&
    (key[1] == '1' || key[1] == '2' || key[1] == '3') &&
    (key[2] == '1' || key[2] == '2' || key[2] == '3');
}

// Usually we have one program per line:
//   XDS
//   XDS VERSION NOVEMBER 3, 2014
//   AIMLESS 0.5.17
// but it can also be a list of programs:
//   autoPROC (Version 1.3.0), AIMLESS, STARANISO
//   autoPROC, XDS (VERSION Jan 26, 2018)
// We assume that:
// - the name has only one word (apologies to Queen of Spades,
//   Force Field X, APEX 2 and Insight II).
// - comma not followed by a digit separates programs
// - brackets and the word VERSION are to be removed from version
inline void add_software(Metadata& meta, SoftwareItem::Classification type,
                         const std::string& name) {
  for (size_t start = 0, end = 0; end != std::string::npos; start = end + 1) {
    end = name.find(',', start);
    while (end != std::string::npos &&
           name[end+1] == ' ' && is_digit(name[end+2]))
      end = name.find(',', end + 1);
    meta.software.emplace_back();
    SoftwareItem& item = meta.software.back();
    item.name = trim_str(name.substr(start, end - start));
    size_t sep = item.name.find(' ');
    if (sep != std::string::npos) {
      size_t ver_start = item.name.find_first_not_of(" (", sep + 1);
      item.version = item.name.substr(ver_start);
      item.name.resize(sep);
      if (!item.version.empty() && item.version.back() == ')')
        item.version.pop_back();
      if (istarts_with(item.version, "version "))
        item.version.erase(0, 8);
    }
    item.classification = type;
    item.pdbx_ordinal = (int) meta.software.size();
  }
}

// REMARK   3   TERM                          COUNT    WEIGHT   FUNCTION.
// REMARK   3    BOND LENGTHS              : 5760   ; 2.000  ; HARMONIC
inline void add_restraint_count_weight(RefinementInfo& ref_info,
                                       const char* key, const char* value) {
  ref_info.restr_stats.emplace_back(key);
  RefinementInfo::Restr& restr = ref_info.restr_stats.back();
  const char* endptr;
  restr.count = no_sign_atoi(value, &endptr);
  if (const char* sep = std::strchr(endptr, ';'))
    restr.weight = simple_atof(sep + 1, &endptr);
  if (const char* sep = std::strchr(endptr, ';'))
    restr.function = read_string(sep+1, 50);
}

inline void read_remark3_line(const char* line, Metadata& meta) {
  // Based on:
  // www.wwpdb.org/documentation/file-format-content/format23/remark3.html
  // and analysis of PDB files.
  // In special cases, such as joint X-ray and neutron refinement 5MOO,
  // PDB file can have two REMARK 3 blocks.
  // Generally, after "REMARK   3" we have either a header-like sentance
  // or a key:value pair with a colon, or a continuation of text from the
  // previous line.
  const char* key_start = skip_blank(line + 10);
  const char* colon = std::strchr(key_start, ':');
  const char* key_end = rtrim_cstr(key_start, colon);
  std::string key(key_start, key_end);
  if (colon) {
    const char* value = skip_blank(colon + 1);
    const char* end = rtrim_cstr(value);
    if (end - value == 4 && std::strncmp(value, "NULL", 4) == 0)
      return;
    if (same_str(key, "PROGRAM"))
      add_software(meta, SoftwareItem::Refinement, std::string(value, end));
    if (meta.refinement.empty())
      return;
    RefinementInfo& ref_info = meta.refinement.back();
    if (same_str(key, "RESOLUTION RANGE HIGH (ANGSTROMS)")) {
      ref_info.resolution_high = simple_atof(value);
    } else if (same_str(key, "RESOLUTION RANGE LOW  (ANGSTROMS)")) {
      ref_info.resolution_low = simple_atof(value);
    } else if (same_str(key, "COMPLETENESS FOR RANGE        (%)")) {
      ref_info.completeness = simple_atof(value);
    } else if (same_str(key, "NUMBER OF REFLECTIONS")) {
      ref_info.reflection_count = std::atoi(value);
    } else if (same_str(key, "CROSS-VALIDATION METHOD")) {
      ref_info.cross_validation_method = std::string(value, end);
    } else if (same_str(key, "FREE R VALUE TEST SET SELECTION")) {
      ref_info.rfree_selection_method = std::string(value, end);
    } else if (same_str(key, "R VALUE     (WORKING + TEST SET)")) {
      ref_info.r_all = simple_atof(value);
    } else if (same_str(key, "R VALUE            (WORKING SET)")) {
      ref_info.r_work = simple_atof(value);
    } else if (same_str(key, "FREE R VALUE")) {
      ref_info.r_free = simple_atof(value);
    } else if (same_str(key, "FREE R VALUE TEST SET COUNT")) {
      ref_info.rfree_set_count = atoi(value);
    } else if (same_str(key, "TOTAL NUMBER OF BINS USED")) {
      ref_info.bin_count = std::atoi(value);
    } else if (same_str(key, "BIN RESOLUTION RANGE HIGH       (A)")) {
      if (!ref_info.bins.empty())
        ref_info.bins.back().resolution_high = simple_atof(value);
    } else if (same_str(key, "BIN RESOLUTION RANGE LOW        (A)")) {
      if (!ref_info.bins.empty())
        ref_info.bins.back().resolution_low = simple_atof(value);
    } else if (same_str(key, "BIN COMPLETENESS (WORKING+TEST) (%)")) {
      if (!ref_info.bins.empty())
        ref_info.bins.back().completeness = simple_atof(value);
    } else if (same_str(key, "REFLECTIONS IN BIN   (WORKING+TEST)")) {
      if (!ref_info.bins.empty())
        ref_info.bins.back().reflection_count = std::atoi(value);
    } else if (same_str(key, "BIN R VALUE          (WORKING+TEST)")) {
      if (!ref_info.bins.empty())
        ref_info.bins.back().r_all = simple_atof(value);
    } else if (same_str(key, "BIN R VALUE           (WORKING SET)")) {
      if (!ref_info.bins.empty())
        ref_info.bins.back().r_work = simple_atof(value);
    } else if (same_str(key, "BIN FREE R VALUE")) {
      if (!ref_info.bins.empty())
        ref_info.bins.back().r_free = simple_atof(value);
    } else if (same_str(key, "BIN FREE R VALUE TEST SET COUNT")) {
      if (!ref_info.bins.empty())
        ref_info.bins.back().rfree_set_count = std::atoi(value);
    } else if (same_str(key, "FROM WILSON PLOT           (A**2)")) {
      // TODO
      // exper.b_wilson = simple_atof(value);
    } else if (same_str(key, "MEAN B VALUE      (OVERALL, A**2)")) {
      ref_info.mean_b = simple_atof(value);
    } else if (same_str(key, "B11 (A**2)")) {
      ref_info.aniso_b[0][0] = simple_atof(value);
    } else if (same_str(key, "B22 (A**2)")) {
      ref_info.aniso_b[1][1] = simple_atof(value);
    } else if (same_str(key, "B33 (A**2)")) {
      ref_info.aniso_b[2][2] = simple_atof(value);
    } else if (same_str(key, "B12 (A**2)")) {
      ref_info.aniso_b[0][1] = simple_atof(value);
    } else if (same_str(key, "B13 (A**2)")) {
      ref_info.aniso_b[0][2] = simple_atof(value);
    } else if (same_str(key, "B23 (A**2)")) {
      ref_info.aniso_b[1][2] = simple_atof(value);
    } else if (same_str(key, "ESD FROM LUZZATI PLOT                    (A)")) {
      ref_info.luzzati_error = simple_atof(value);
    } else if (same_str(key, "DPI (BLOW EQ-10) BASED ON R VALUE        (A)")) {
      ref_info.dpi_blow_r = simple_atof(value);
    } else if (same_str(key, "DPI (BLOW EQ-9) BASED ON FREE R VALUE    (A)")) {
      ref_info.dpi_blow_rfree = simple_atof(value);
    } else if (same_str(key, "DPI (CRUICKSHANK) BASED ON R VALUE       (A)")) {
      ref_info.dpi_cruickshank_r = simple_atof(value);
    } else if (same_str(key, "DPI (CRUICKSHANK) BASED ON FREE R VALUE  (A)")) {
      ref_info.dpi_cruickshank_rfree = simple_atof(value);
    } else if (same_str(key, "CORRELATION COEFFICIENT FO-FC")) {
      ref_info.cc_fo_fc = simple_atof(value);
    } else if (same_str(key, "CORRELATION COEFFICIENT FO-FC FREE")) {
      ref_info.cc_fo_fc_free = simple_atof(value);
    } else if (same_str(key, "BOND LENGTHS")) {
      add_restraint_count_weight(ref_info, "t_bond_d", value);
    } else if (same_str(key, "BOND ANGLES")) {
      add_restraint_count_weight(ref_info, "t_angle_deg", value);
    } else if (same_str(key, "TORSION ANGLES")) {
      add_restraint_count_weight(ref_info, "t_dihedral_angle_d", value);
    } else if (same_str(key, "TRIGONAL CARBON PLANES")) {
      add_restraint_count_weight(ref_info, "t_trig_c_planes", value);
    } else if (same_str(key, "GENERAL PLANES")) {
      add_restraint_count_weight(ref_info, "t_gen_planes", value);
    } else if (same_str(key, "ISOTROPIC THERMAL FACTORS")) {
      add_restraint_count_weight(ref_info, "t_it", value);
    } else if (same_str(key, "BAD NON-BONDED CONTACTS")) {
      add_restraint_count_weight(ref_info, "t_nbd", value);
    } else if (same_str(key, "IMPROPER TORSIONS")) {
      add_restraint_count_weight(ref_info, "t_improper_torsion", value);
    } else if (same_str(key, "CHIRAL IMPROPER TORSION")) {
      add_restraint_count_weight(ref_info, "t_chiral_improper_torsion", value);
    } else if (same_str(key, "SUM OF OCCUPANCIES")) {
      add_restraint_count_weight(ref_info, "t_sum_occupancies", value);
    } else if (same_str(key, "UTILITY DISTANCES")) {
      add_restraint_count_weight(ref_info, "t_utility_distance", value);
    } else if (same_str(key, "UTILITY ANGLES")) {
      add_restraint_count_weight(ref_info, "t_utility_angle", value);
    } else if (same_str(key, "UTILITY TORSION")) {
      add_restraint_count_weight(ref_info, "t_utility_torsion", value);
    } else if (same_str(key, "IDEAL-DIST CONTACT TERM")) {
      add_restraint_count_weight(ref_info, "t_ideal_dist_contact", value);
    } else if (same_str(key, "BOND LENGTHS                       (A)")) {
      impl::find_or_add(ref_info.restr_stats, "t_bond_d").dev_ideal
        = read_double(value, 50);
    } else if (same_str(key, "BOND ANGLES                  (DEGREES)")) {
      impl::find_or_add(ref_info.restr_stats, "t_angle_deg").dev_ideal
        = read_double(value, 50);
{'fraction_non_alphanumeric': 0.15626236644242184, 'fraction_numerical': 0.012366442421844084, 'mean_word_length': 3.216266944734098, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '87342', 'n_tokens_mistral': 7853, 'n_tokens_neox': 7527, 'n_words': 1683}