<commit_before>#!/usr/bin/env python

from collections import OrderedDict
import inspect
import json
import os
import re
import shutil
import io
from subprocess import call, Popen, PIPE
import sys, getopt
import pkg_resources
import subprocess

from jinja2 import Environment, FileSystemLoader

from drafter_postprocessing.json_processing import postprocess_drafter_json
from apib_extra_parse_utils import preprocess_apib_parameters_lines, start_apib_section, get_indentation


def print_api_spec_title_to_extra_file(input_file_path, extra_sections_file_path):
    """Extracts the title of the API specification and writes it to the extra sections file.

    Arguments:
    input_file_path -- File with the API specification
    extra_sections_file_path -- File where we will write the extra sections
    """
    with open(input_file_path, 'rU') as input_file_path, open(extra_sections_file_path, 'w') as extra_sections_file:
        line = input_file_path.readline()
        while (line != "" and not line.startswith("# ")):
            line = input_file_path.readline()
    
        extra_sections_file.write( line )


def separate_extra_sections_and_api_blueprint(input_file_path, extra_sections_file_path, API_blueprint_file_path):
    """Divides a Fiware API specification into extra sections and its API blueprint.

    Arguments:
    input_file_path -- A Fiware API specification file.
    extra_sections_file_path -- Resulting file containing extra information about the API specification.
    API_blueprint_file_path -- Resulting file containing the API blueprint of the Fiware API.
    """
    print_api_spec_title_to_extra_file(input_file_path, extra_sections_file_path) 

    with open(input_file_path, 'rU') as input_file, open(extra_sections_file_path, 'a') as extra_sections_file, open(API_blueprint_file_path, 'w') as API_blueprint_file:
        
        line_counter = 0
        title_line_end = -1
        apib_line_start = -1

        metadata_section = True
        apib_part = False
        title_section = False
        parameters_section = False
        data_structures_section = 0

        for line in input_file:
            line_counter += 1

            copy = False

            if metadata_section and len(line.split(':')) == 1:
                metadata_section = False
                title_section = True
            
            if metadata_section:
                copy = False
            else:
                if title_section and line.startswith('##'):
                    title_section = False

                if title_section:
                    copy = False

                else:
                    if not apib_part:
                        apib_part = start_apib_section(line)
                        if title_line_end < 0:
                            title_line_end = line_counter
                        
                    if not apib_part:
                        copy = True
                    else:
                        copy = False
                        if apib_line_start < 0:
                            apib_line_start = line_counter

            if copy:
                extra_sections_file.write(line)
            else:
                line = line.replace('\t','    ')
                (line, parameters_section, data_structures_section) = preprocess_apib_parameters_lines(line, parameters_section, 
                                                    data_structures_section)
                API_blueprint_file.write(line)

    return (title_line_end, apib_line_start)


def convert_message_error_lines(drafter_output, title_line_end, apib_line_start):
    """Convert the error lines to match the extended FIWARE APIB file format
<commit_msg>Avoid displaying API REST title under abstract<commit_after>#!/usr/bin/env python

from collections import OrderedDict
import inspect
import json
import os
import re
import shutil
import io
from subprocess import call, Popen, PIPE
import sys, getopt
import pkg_resources
import subprocess

from jinja2 import Environment, FileSystemLoader

from drafter_postprocessing.json_processing import postprocess_drafter_json
from apib_extra_parse_utils import preprocess_apib_parameters_lines, start_apib_section, get_indentation


def print_api_spec_title_to_extra_file(input_file_path, extra_sections_file_path):
    """Extracts the title of the API specification and writes it to the extra sections file.

    Arguments:
    input_file_path -- File with the API specification
    extra_sections_file_path -- File where we will write the extra sections
    """
    with open(input_file_path, 'rU') as input_file_path, open(extra_sections_file_path, 'w') as extra_sections_file:
        line = input_file_path.readline()
        while (line != "" and not line.startswith("# ")):
            line = input_file_path.readline()
    
        extra_sections_file.write( line )


def separate_extra_sections_and_api_blueprint(input_file_path, extra_sections_file_path, API_blueprint_file_path):
    """Divides a Fiware API specification into extra sections and its API blueprint.

    Arguments:
    input_file_path -- A Fiware API specification file.
    extra_sections_file_path -- Resulting file containing extra information about the API specification.
    API_blueprint_file_path -- Resulting file containing the API blueprint of the Fiware API.
    """
    print_api_spec_title_to_extra_file(input_file_path, extra_sections_file_path) 

    with open(input_file_path, 'rU') as input_file, open(extra_sections_file_path, 'a') as extra_sections_file, open(API_blueprint_file_path, 'w') as API_blueprint_file:
        
        line_counter = 0
        title_line_end = -1
        apib_line_start = -1

        metadata_section = True
        apib_part = False
        title_section = False
        parameters_section = False
        data_structures_section = 0

        for line in input_file:
            line_counter += 1

            copy = False

            if metadata_section and len(line.split(':')) == 1:
                metadata_section = False
                title_section = True
            
            if metadata_section:
                copy = False
            else:
                if title_section and line.startswith('##'):
                    title_section = False

                if title_section:
                    copy = False

                else:
                    if not apib_part:
                        apib_part = start_apib_section(line)
                        if title_line_end < 0:
                            title_line_end = line_counter
                        
                    if not apib_part:
                        copy = True
                    else:
                        copy = False
                        if apib_line_start < 0:
                            if line.strip() == "# REST API":
                                line = '\n'
                            apib_line_start = line_counter

            if copy:
                extra_sections_file.write(line)
            else:
                line = line.replace('\t','    ')
                (line, parameters_section, data_structures_section) = preprocess_apib_parameters_lines(line, parameters_section, 
                                                    data_structures_section)
                API_blueprint_file.write(line)

    return (title_line_end, apib_line_start)


def convert_message_error_lines(drafter_output, title_line_end, apib_line_start):
    """Convert the error lines to match the extended FIWARE APIB file format

{'fraction_non_alphanumeric': 0.05577636459989401, 'fraction_numerical': 0.0023847376788553257, 'mean_word_length': 2.163872590108969, 'pattern_counts': {'":': 1, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3008452', 'n_tokens_mistral': 2079, 'n_tokens_neox': 1960, 'n_words': 538}