<commit_before>#lang racket/base

#|
   Copyright 2016-2017 <NAME>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
|#

(provide (all-defined-out))
(require "lib.rkt"
         "constants.rkt"
         ffi/unsafe
         (for-syntax racket/base
                     syntax/parse
                     racket/syntax))

(define-syntax (define-ffmpeg-cstruct stx)
  (syntax-parse stx
    [(_ id ([field-name field-rest ...] ...) rest ...)
     #:with id-field-names (format-id stx "~a-field-names" #'id)
     (quasisyntax/loc stx
       (begin
         #,(quasisyntax/loc stx
             (define id-field-names '#,#'(field-name ...)))
         #,(syntax/loc stx
             (define-cstruct id ([field-name field-rest ...] ...) rest ...))))]))

(define-cstruct _av-dictionary-entry
  ([key _string]
   [value _string]))

(define-cstruct _av-dictionary
  ([count _int]
   [elems _av-dictionary-entry-pointer]))

(define-cstruct _avclass
  ([class-name _string]
   [item-name _fpointer]
   [option _pointer]
   [version _int]
   [log-level-offset-offset _int]
   [parent-log-context-offset _int]
   [child-next _pointer]
   [catagory _avclass-category]
   [get-catagory _fpointer]
   [query-range _fpointer]))

(define-cstruct _avio-context
  ([buffer _bytes]
   [buffer-size _int]
   [buf-ptr _bytes]
   [buf-end _bytes]
   [opeque _pointer]
   [read-packet _fpointer]
   [write-packet _fpointer]
   [seek _fpointer]
   [pos _int64]
   [must-flush _int]
   [eof-reached _int]
   [write-flag _int]
   [max-packet-size _int]
   [checksum _ulong]
   [checksum-ptr _pointer]
   [update-checksum _fpointer]
   [error _int]
   [read-pause _fpointer]
   [read-seek _fpointer]
   [seekable _int]
   [maxsize _int64]
   [direct _int]
   [bytes-read _int64]
   [seek-count _int]
   [writeout-count _int]
   [orig-buffer-size _int]))

(define-cstruct _avio-interrupt-cb
  ([callback _fpointer]
   [opaque _pointer]))

(define-cstruct _av-input-format
  ([name _string]
   [long-name _string]
   [flags _avformat-flags]
   [extensions _string]
   [codec-tag _pointer]
   [priv-class _pointer]
   [mime-type _string]
   [next _pointer]
   [raw-codec-id _int]
   [priv-data-size _int]
   [read-probe _fpointer]
   [read-header _fpointer]
   [read-packet _fpointer]
   [read-close _fpointer]
   [read-seek _fpointer]
   [read-timestamp _fpointer]
   [read-play _fpointer]
   [read-pause _fpointer]
   [read-seek2 _fpointer]
   [get-device-list _fpointer]
   [create-device-capabilities _fpointer]
   [free-device-capabilities _fpointer]))

(define-cstruct _av-output-format
  ([name _string]
   [long-name _string]
   [mime-type _string]
   [extensions _string]
   [audio-codec _avcodec-id]
   [video-codec _avcodec-id]
   [subtitle-codec _avcodec-id]
   [flags _avformat-flags]
   [codec-tag _pointer]
   [priv-class _pointer]))

(define-ffmpeg-cstruct _avformat-context
  ([av-class _pointer]
   [iformat _av-input-format-pointer/null]
   [oformat _av-output-format-pointer/null]
   [priv_data _pointer]
   [pb _avio-context-pointer/null]
   [ctx-flags _int] ; _avformat-context-flags
   [nb-streams _uint]
   [streams-data _pointer]
   [filename (_array _byte 1024)]
   [start-time _int64]
   [duration _int64]
   [bit-rate _int]
   [packet-size _uint]
   [max-delay _uint]
   [flags _avformat-flags]
   [probesize _uint]
   [max-anlayze-duration _int]
   [key _pointer]
   [keylen _int]
   [nb-programs _uint]
   [programs _pointer]
   [video-codec-id _avcodec-id]
   [audio-codec-id _avcodec-id]
   [subtitle-codec-id _avcodec-id]
   [max-index-size _uint]
   [max-picture-buffer _uint]
   [nb-chapters _uint]
   [chapters _pointer]
   [metadata _pointer]
   [start-time-realtime _int64]
   [fps-probe-size _int]
   [error-recognition _int]
   [interrupt-callback _avio-interrupt-cb]
   [debug _int]
   [max-interleave-delay _int64]
   [string-std-compliance _int]
   [event-flags _int]
   [max-ts-probe _int]
   [avoid-negative-ts _int]
   [ts-id _int]
   [audio-preload _int]
   [max-chunk-duration _int]
   [max-chunk-size _int]
   [use-wallclock-as-timestamps _int]
   [avio-flags _int]
   [durration-estimation-method _av-duration-estimation-method]
   [skip-initial-bytes _int64]
   [correct-ts-overflow _uint]
   [seek2any _int]
   [flush-packets _int]
   [probe-score _int]
   [format-probesize _int]
   [codec-whitelist _pointer]
   [format-whitelist _pointer]
   [internal _pointer]
   [io-repositioned _int]
   [video-codec _pointer]
   [audio-codec _pointer]
   [subtitle-codec _pointer]
   [data-codec _pointer]
   [metadata-header-padding _int]
   [opaque _pointer]
   [control-message-cb _pointer]
   [output-ts-offset _int64]
   [max-analyze-duration2 _int64]
   [probesize2 _int64]
   [dump-separator _uint8]
   [data-codec-id _avcodec-id]
   [open-cb _pointer]))
(define (avformat-context-streams v)
  (cblock->list (avformat-context-streams-data v)
                _avstream-pointer
                (avformat-context-nb-streams v)))

(define _avrational
  (let ()
    (define-cstruct _avrational
      ([num _int]
       [den _int]))
    (make-ctype _avrational
                (λ (x)
                  (make-avrational (numerator x)
                                   (denominator x)))
                (λ (x)
                  (when (= (avrational-den x) 0)
                    (error 'avrational "Invalid AVRational ~a/~a"
                           (avrational-num x)
                           (avrational-den x)))
                  (/ (avrational-num x)
                     (avrational-den x))))))

(define-cstruct _avbuffer-ref
  ([buffer _pointer]
   [data _pointer]
   [size _int]))
<commit_msg>Add chapters datatype<commit_after>#lang racket/base

#|
   Copyright 2016-2017 <NAME>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
|#

(provide (all-defined-out))
(require "lib.rkt"
         "constants.rkt"
         ffi/unsafe
         (for-syntax racket/base
                     syntax/parse
                     racket/syntax))

(define-syntax (define-ffmpeg-cstruct stx)
  (syntax-parse stx
    [(_ id ([field-name field-rest ...] ...) rest ...)
     #:with id-field-names (format-id stx "~a-field-names" #'id)
     (quasisyntax/loc stx
       (begin
         #,(quasisyntax/loc stx
             (define id-field-names '#,#'(field-name ...)))
         #,(syntax/loc stx
             (define-cstruct id ([field-name field-rest ...] ...) rest ...))))]))

(define-cstruct _av-dictionary-entry
  ([key _string]
   [value _string]))

(define-cstruct _av-dictionary
  ([count _int]
   [elems _av-dictionary-entry-pointer]))

(define-cstruct _avclass
  ([class-name _string]
   [item-name _fpointer]
   [option _pointer]
   [version _int]
   [log-level-offset-offset _int]
   [parent-log-context-offset _int]
   [child-next _pointer]
   [catagory _avclass-category]
   [get-catagory _fpointer]
   [query-range _fpointer]))

(define-cstruct _avio-context
  ([buffer _bytes]
   [buffer-size _int]
   [buf-ptr _bytes]
   [buf-end _bytes]
   [opeque _pointer]
   [read-packet _fpointer]
   [write-packet _fpointer]
   [seek _fpointer]
   [pos _int64]
   [must-flush _int]
   [eof-reached _int]
   [write-flag _int]
   [max-packet-size _int]
   [checksum _ulong]
   [checksum-ptr _pointer]
   [update-checksum _fpointer]
   [error _int]
   [read-pause _fpointer]
   [read-seek _fpointer]
   [seekable _int]
   [maxsize _int64]
   [direct _int]
   [bytes-read _int64]
   [seek-count _int]
   [writeout-count _int]
   [orig-buffer-size _int]))

(define-cstruct _avio-interrupt-cb
  ([callback _fpointer]
   [opaque _pointer]))

(define-cstruct _av-input-format
  ([name _string]
   [long-name _string]
   [flags _avformat-flags]
   [extensions _string]
   [codec-tag _pointer]
   [priv-class _pointer]
   [mime-type _string]
   [next _pointer]
   [raw-codec-id _int]
   [priv-data-size _int]
   [read-probe _fpointer]
   [read-header _fpointer]
   [read-packet _fpointer]
   [read-close _fpointer]
   [read-seek _fpointer]
   [read-timestamp _fpointer]
   [read-play _fpointer]
   [read-pause _fpointer]
   [read-seek2 _fpointer]
   [get-device-list _fpointer]
   [create-device-capabilities _fpointer]
   [free-device-capabilities _fpointer]))

(define-cstruct _av-output-format
  ([name _string]
   [long-name _string]
   [mime-type _string]
   [extensions _string]
   [audio-codec _avcodec-id]
   [video-codec _avcodec-id]
   [subtitle-codec _avcodec-id]
   [flags _avformat-flags]
   [codec-tag _pointer]
   [priv-class _pointer]))

(define-ffmpeg-cstruct _avchapter
  ([id _int]
   [time-base _avrational]
   [start _int64]
   [end _int64]
   [metadata _av-dictionary-pointer/null]))

(define-ffmpeg-cstruct _avformat-context
  ([av-class _pointer]
   [iformat _av-input-format-pointer/null]
   [oformat _av-output-format-pointer/null]
   [priv_data _pointer]
   [pb _avio-context-pointer/null]
   [ctx-flags _int] ; _avformat-context-flags
   [nb-streams _uint]
   [streams-data _pointer]
   [filename (_array _byte 1024)]
   [start-time _int64]
   [duration _int64]
   [bit-rate _int]
   [packet-size _uint]
   [max-delay _uint]
   [flags _avformat-flags]
   [probesize _uint]
   [max-anlayze-duration _int]
   [key _pointer]
   [keylen _int]
   [nb-programs _uint]
   [programs _pointer]
   [video-codec-id _avcodec-id]
   [audio-codec-id _avcodec-id]
   [subtitle-codec-id _avcodec-id]
   [max-index-size _uint]
   [max-picture-buffer _uint]
   [nb-chapters _uint]
   [chapters-data _pointer]
   [metadata _pointer]
   [start-time-realtime _int64]
   [fps-probe-size _int]
   [error-recognition _int]
   [interrupt-callback _avio-interrupt-cb]
   [debug _int]
   [max-interleave-delay _int64]
   [string-std-compliance _int]
   [event-flags _int]
   [max-ts-probe _int]
   [avoid-negative-ts _int]
   [ts-id _int]
   [audio-preload _int]
   [max-chunk-duration _int]
   [max-chunk-size _int]
   [use-wallclock-as-timestamps _int]
   [avio-flags _int]
   [durration-estimation-method _av-duration-estimation-method]
   [skip-initial-bytes _int64]
   [correct-ts-overflow _uint]
   [seek2any _int]
   [flush-packets _int]
   [probe-score _int]
   [format-probesize _int]
   [codec-whitelist _pointer]
   [format-whitelist _pointer]
   [internal _pointer]
   [io-repositioned _int]
   [video-codec _pointer]
   [audio-codec _pointer]
   [subtitle-codec _pointer]
   [data-codec _pointer]
   [metadata-header-padding _int]
   [opaque _pointer]
   [control-message-cb _pointer]
   [output-ts-offset _int64]
   [max-analyze-duration2 _int64]
   [probesize2 _int64]
   [dump-separator _uint8]
   [data-codec-id _avcodec-id]
   [open-cb _pointer]))
(define (avformat-context-streams v)
  (cblock->list (avformat-context-streams-data v)
                _avstream-pointer
                (avformat-context-nb-streams v)))
(define (avformat-context-chapters v)
  (cblock->list (avformat-context-chapters-data v)
                _avchapter-pointer
                (avformat-context-nb-chapters v)))

(define _avrational
  (let ()
    (define-cstruct _avrational
      ([num _int]
       [den _int]))
    (make-ctype _avrational
                (λ (x)
                  (make-avrational (numerator x)
                                   (denominator x)))
                (λ (x)
                  (when (= (avrational-den x) 0)
                    (error 'avrational "Invalid AVRational ~a/~a"
                           (avrational-num x)
                           (avrational-den x)))
                  (/ (avrational-num x)
                     (avrational-den x))))))

(define-cstruct _avbuffer-ref
  ([buffer _pointer]
   [data _pointer]
   [size _int]))

{'fraction_non_alphanumeric': 0.12319936331078392, 'fraction_numerical': 0.007321925984878631, 'mean_word_length': 2.691539365452409, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4014484', 'n_tokens_mistral': 4604, 'n_tokens_neox': 4424, 'n_words': 987}