<commit_before>header: |
    (?P<intro>
        \*\ Sea-Bird\ SBE\ .*\ Data\ File:\r?\n
        (
            \*\ .*\n  # Antything starting with *
        )+
    )
    (?P<notes>
        (
            \*\* .*\n  # Antything starting with *
        )*
    )
    # temporary solution just to run while I'm sailing
    # The notes not necessarily are after the intro, they
    # can be in the middle of the intro.
        (
            \*\ .*\n  # Antything starting with *
        )*
    \s*
    (?P<descriptors>
        (
            \#.*\n  # Antything
        )+
    )

sep: |
    \*END\*\r\n

data: |
  (?P<data>
    (?:
        (?:                     # Start of one variable
          \ *
          -?                    # Can be a negative number
          \d+                   # Any digits
            (?:
              \.\d+               # Real numbers
              (?:
                e[\+|-]\d+        # Cientific notation
              )?
            )?
        )+                      # Repeat as much variables as there are
        #  (?:
        #  )?
          \s*                   # Following empty spaces, end of line
    )+                          # One or more lines
  )

# First section of the header
intro:
    sbe_model:
       \*\ Sea-Bird\ SBE\ +(?P<value>\d.*)\ Data\ File:\r?\n
    LATITUDE: |
        \*\ NMEA\ Latitude\ =\ (?P<value>.*?)\r?\n
    LONGITUDE: |
        \*\ NMEA\ Longitude\ =\ (?P<value>.*?)\r?\n
    gps_datetime: |
        \*\ NMEA\ UTC\ \(Time\)\ =\ (?P<value>.*?)\r?\n
    seasave: |
        \*\ Software\ Version\ Seasave\ (?P<value>.*?)\r?\n

# Third section of the header
descriptors:
    nquan: |
      \#\ nquan\ =\ (?P<value>\d+)
    nvalues: |
      \#\ nvalues\ =\ (?P<value>\d+)
    start_time: |
      \#\ start_time\ =\ (?P<value>.*?)\r\n      # Should work, but I'm not happy with this.
    bad_flag: |
      \#\ bad_flag\ =\ (?P<value>.*?)\s
    file_type: |
      \#\ file_type\ \=\ (?P<value>.*?)\r\n 
    awkward_xml: |
      (?P<value>
        \#\ \<\?xml\ version="\d+\.\d+"\ encoding="UTF-8"\?>\r\n
        (?:\#\s+<.*>\r\n)+
      )

# <?xml version="1.0" encoding="UTF-8"?>
#      # # <Sensors count="7" >
#      # #   <sensor Channel="1" >
#      # #     <!-- Frequency 0, Temperature -->

#descriptors: |
#    \#\ nquan\ =\ (?P<nquan>\d+) \s*
#    \#\ nvalues\ =\ (?P<nvalues>\d+) \s*
#    \#\ units\ =\ \w*\s*
#    (?P<names>
#      (
#      \#\ name\ \d+\ =\ (?P<name>\w+) #\: Time, Elapsed [seconds]
#      .*
#      \s*
#      )+
#    )

fieldname: |
  \#\ name\ (?P<id>\d+)\ =\ (?P<name>.*)\:(?:\ (?P<longname>.*))?\r\s

fieldspan: |
  \#\ span\ (?P<id>\d+)\ =\ (?P<valuemin>.*),\ (?P<valuemax>.*)\r\s

LATITUDE: |
    # [LAT|LATITUDE]
        (?:Latitude:?)?
        \ *
    (?P<degree>\d{2})
    \ +
    (?P<minute>\d{2}
        [\.|\,]
        \d{1,3})
    \ *
    (?P<hemisphere>[N|n|S|s])

LONGITUDE: |
    # [LON|LONG|LONGITUDE]
        (?:Latitude:?)?
        \ *
    (?P<degree>\d{2,3})
    \ +
    (?P<minute>\d{2}
        [\.|\,]
        \d{1,3})
    \ *
    (?P<hemisphere>[W|w|E|e])
<commit_msg>BUGFIX #20: Alternative header<commit_after>header: |
    (?P<intro>
        \*\ Sea-Bird\ SBE\ ?.*\ Data\ File:\r?\n
        (
            \*\ .*\n  # Antything starting with *
        )+
    )
    (?P<notes>
        (
            \*\* .*\n  # Antything starting with *
        )*
    )
    # temporary solution just to run while I'm sailing
    # The notes not necessarily are after the intro, they
    # can be in the middle of the intro.
        (
            \*\ .*\n  # Antything starting with *
        )*
    \s*
    (?P<descriptors>
        (
            \#.*\n  # Antything
        )+
    )

sep: |
    \*END\*\r\n

data: |
  (?P<data>
    (?:
        (?:                     # Start of one variable
          \ *
          -?                    # Can be a negative number
          \d+                   # Any digits
            (?:
              \.\d+               # Real numbers
              (?:
                e[\+|-]\d+        # Cientific notation
              )?
            )?
        )+                      # Repeat as much variables as there are
        #  (?:
        #  )?
          \s*                   # Following empty spaces, end of line
    )+                          # One or more lines
  )

# First section of the header
intro:
    sbe_model:
       \*\ Sea-Bird\ SBE\ +(?P<value>\d.*)\ Data\ File:\r?\n
    LATITUDE: |
        \*\ NMEA\ Latitude\ =\ (?P<value>.*?)\r?\n
    LONGITUDE: |
        \*\ NMEA\ Longitude\ =\ (?P<value>.*?)\r?\n
    gps_datetime: |
        \*\ NMEA\ UTC\ \(Time\)\ =\ (?P<value>.*?)\r?\n
    seasave: |
        \*\ Software\ Version\ Seasave\ (?P<value>.*?)\r?\n

# Third section of the header
descriptors:
    nquan: |
      \#\ nquan\ =\ (?P<value>\d+)
    nvalues: |
      \#\ nvalues\ =\ (?P<value>\d+)
    start_time: |
      \#\ start_time\ =\ (?P<value>.*?)\r\n      # Should work, but I'm not happy with this.
    bad_flag: |
      \#\ bad_flag\ =\ (?P<value>.*?)\s
    file_type: |
      \#\ file_type\ \=\ (?P<value>.*?)\r\n 
    awkward_xml: |
      (?P<value>
        \#\ \<\?xml\ version="\d+\.\d+"\ encoding="UTF-8"\?>\r\n
        (?:\#\s+<.*>\r\n)+
      )

# <?xml version="1.0" encoding="UTF-8"?>
#      # # <Sensors count="7" >
#      # #   <sensor Channel="1" >
#      # #     <!-- Frequency 0, Temperature -->

#descriptors: |
#    \#\ nquan\ =\ (?P<nquan>\d+) \s*
#    \#\ nvalues\ =\ (?P<nvalues>\d+) \s*
#    \#\ units\ =\ \w*\s*
#    (?P<names>
#      (
#      \#\ name\ \d+\ =\ (?P<name>\w+) #\: Time, Elapsed [seconds]
#      .*
#      \s*
#      )+
#    )

fieldname: |
  \#\ name\ (?P<id>\d+)\ =\ (?P<name>.*)\:(?:\ (?P<longname>.*))?\r\s

fieldspan: |
  \#\ span\ (?P<id>\d+)\ =\ (?P<valuemin>.*),\ (?P<valuemax>.*)\r\s

LATITUDE: |
    # [LAT|LATITUDE]
        (?:Latitude:?)?
        \ *
    (?P<degree>\d{2})
    \ +
    (?P<minute>\d{2}
        [\.|\,]
        \d{1,3})
    \ *
    (?P<hemisphere>[N|n|S|s])

LONGITUDE: |
    # [LON|LONG|LONGITUDE]
        (?:Latitude:?)?
        \ *
    (?P<degree>\d{2,3})
    \ +
    (?P<minute>\d{2}
        [\.|\,]
        \d{1,3})
    \ *
    (?P<hemisphere>[W|w|E|e])

{'fraction_non_alphanumeric': 0.2323966408268734, 'fraction_numerical': 0.005490956072351421, 'mean_word_length': 1.2454677302393038, 'pattern_counts': {'":': 0, '<': 77, '<?xml version=': 2, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7354373', 'n_tokens_mistral': 2560, 'n_tokens_neox': 2408, 'n_words': 411}