<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# This file is part of Export Layers.
#
# Copyright (C) 2013-2016 khalim19 <<EMAIL>>
#
# Export Layers is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Export Layers is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Export Layers.  If not, see <http://www.gnu.org/licenses/>.
#

"""
This script creates a ZIP package for releases from the plug-in source.

This script requires the `pathspec` library (for matching files by patterns):
https://github.com/cpburnz/python-path-specification
"""

from __future__ import absolute_import, division, print_function, unicode_literals
import export_layers.pygimplib as pygimplib
from future.builtins import *

import io
import os
import re
import shutil
import subprocess
import tempfile
import sys
import zipfile

import pathspec

import export_layers.config
from export_layers.pygimplib import pgconstants
from export_layers.pygimplib import pgpath
from export_layers.pygimplib import pgutils

#===============================================================================

export_layers.config.init()

pygimplib.init()

RESOURCES_PATH = os.path.dirname(pgutils.get_current_module_file_path())
PLUGINS_PATH = os.path.dirname(RESOURCES_PATH)

OUTPUT_FILENAME_PREFIX = "export-layers"
OUTPUT_FILENAME_EXTENSION = "zip"

INCLUDE_LIST_FILENAME = "./make_package_included_files.txt"

FILENAMES_TO_RENAME = {
  "README.md": "Readme.txt",
  "CHANGELOG.md": "Changelog.txt",
  "Readme for Translators.md": "Readme for Translators.txt",
}

NUM_LEADING_SPACES_TO_TRIM = 4

#===============================================================================


def _trim_leading_spaces(file_to_read, file_to_write, num_leading_spaces):
  line = file_to_read.readline()
  while line:
    if not line.strip():
      # Write back the empty/whitespace-only line.
      file_to_write.write(line)
      
      # Trim leading spaces from subsequent lines.
      line = file_to_read.readline()
      while line.startswith(" " * num_leading_spaces):
        file_to_write.write(line.lstrip(" "))
        line = file_to_read.readline()
    else:
      file_to_write.write(line)
      line = file_to_read.readline()


def _rename_filenames_inside_file(file_to_read, file_to_write, filenames_to_rename):
  """
  Rename the filenames inside the file that were or will be renamed using the
  `filenames_to_rename` dict.
  """
  
  filenames_to_rename_patterns = {}
  for src_filename, dest_filename in filenames_to_rename.items():
    src_filename_pattern = re.escape(src_filename).replace("\\ ", "\\s")
    filenames_to_rename_patterns[src_filename_pattern] = dest_filename
  
  for line in file_to_read:
    processed_line = line
    for src_filename_pattern, dest_filename in filenames_to_rename_patterns.items():
      processed_line = re.sub(
        src_filename_pattern, lambda match: dest_filename, processed_line)
    file_to_write.write(processed_line)


def _remove_download_link(file_to_read, file_to_write):
  line = file_to_read.readline()
  while line:
    if "download latest release" not in line.lower():
      file_to_write.write(line)
    else:
      break
    line = file_to_read.readline()
  
  # Skip the download link
  line = file_to_read.readline()
  # Skip empty line
  line = file_to_read.readline()
  
  while line:
    file_to_write.write(line)
    line = file_to_read.readline()


def process_file(filename, *process_functions_and_args):
  
  def _prepare_files(file_to_read, file_to_write):
    file_to_read.seek(0)
    file_to_write.seek(0)
    file_to_write.truncate()
  
  temp_dir = tempfile.mkdtemp()
  temp_filename_copy = os.path.join(temp_dir, "temp")
  shutil.copy2(filename, temp_filename_copy)
  
  temp_file_copy = io.open(temp_filename_copy, "r+b")
  temp_file = tempfile.NamedTemporaryFile("r+", dir=temp_dir, delete=False)
  
  last_modified_filename = None
  file_to_read = temp_file_copy
  file_to_write = temp_file
  for function_and_args in process_functions_and_args:
    _prepare_files(file_to_read, file_to_write)
    
    process_function = function_and_args[0]
    process_function_additional_args = function_and_args[1:]
    process_function(file_to_read, file_to_write, *process_function_additional_args)
    
    last_modified_filename = file_to_write.name
    file_to_read, file_to_write = file_to_write, file_to_read
  
  temp_file_copy.close()
  temp_file.close()
  
  shutil.copy2(last_modified_filename, filename)
  
  os.remove(temp_filename_copy)
  os.remove(temp_file.name)
  os.rmdir(temp_dir)


#===============================================================================

FILES_TO_PROCESS = {
  FILENAMES_TO_RENAME["README.md"]: [
    (_trim_leading_spaces, NUM_LEADING_SPACES_TO_TRIM),
    (_rename_filenames_inside_file, FILENAMES_TO_RENAME),
    (_remove_download_link,)
  ]
}
<commit_msg>make_package: Use text mode for temporary files<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# This file is part of Export Layers.
#
# Copyright (C) 2013-2016 khalim19 <<EMAIL>>
#
# Export Layers is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Export Layers is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Export Layers.  If not, see <http://www.gnu.org/licenses/>.
#

"""
This script creates a ZIP package for releases from the plug-in source.

This script requires the `pathspec` library (for matching files by patterns):
https://github.com/cpburnz/python-path-specification
"""

from __future__ import absolute_import, division, print_function, unicode_literals
import export_layers.pygimplib as pygimplib
from future.builtins import *

import io
import os
import re
import shutil
import subprocess
import tempfile
import sys
import zipfile

import pathspec

import export_layers.config
from export_layers.pygimplib import pgconstants
from export_layers.pygimplib import pgpath
from export_layers.pygimplib import pgutils

#===============================================================================

export_layers.config.init()

pygimplib.init()

RESOURCES_PATH = os.path.dirname(pgutils.get_current_module_file_path())
PLUGINS_PATH = os.path.dirname(RESOURCES_PATH)

OUTPUT_FILENAME_PREFIX = "export-layers"
OUTPUT_FILENAME_EXTENSION = "zip"

INCLUDE_LIST_FILENAME = "./make_package_included_files.txt"

FILENAMES_TO_RENAME = {
  "README.md": "Readme.txt",
  "CHANGELOG.md": "Changelog.txt",
  "Readme for Translators.md": "Readme for Translators.txt",
}

NUM_LEADING_SPACES_TO_TRIM = 4

#===============================================================================


def process_file(filename, *process_functions_and_args):
  
  def _prepare_files(file_to_read, file_to_write):
    file_to_read.seek(0)
    file_to_write.seek(0)
    file_to_write.truncate()
  
  def _create_temp_file(temp_dir, mode, encoding):
    temp_file = tempfile.NamedTemporaryFile(mode, dir=temp_dir, delete=False)
    temp_file.close()
    temp_file = io.open(temp_file.name, mode, encoding=encoding)
    
    return temp_file
  
  temp_dir = tempfile.mkdtemp()
  temp_filename_copy = os.path.join(temp_dir, "temp")
  shutil.copy2(filename, temp_filename_copy)
  
  temp_file_copy = io.open(
    temp_filename_copy, "r+", encoding=pgconstants.TEXT_FILE_CHARACTER_ENDOCING)
  temp_file = _create_temp_file(temp_dir, "r+", pgconstants.TEXT_FILE_CHARACTER_ENDOCING)
  
  last_modified_filename = None
  file_to_read = temp_file_copy
  file_to_write = temp_file
  for function_and_args in process_functions_and_args:
    _prepare_files(file_to_read, file_to_write)
    
    process_function = function_and_args[0]
    process_function_additional_args = function_and_args[1:]
    process_function(file_to_read, file_to_write, *process_function_additional_args)
    
    last_modified_filename = file_to_write.name
    file_to_read, file_to_write = file_to_write, file_to_read
  
  temp_file_copy.close()
  temp_file.close()
  
  shutil.copy2(last_modified_filename, filename)
  
  os.remove(temp_filename_copy)
  os.remove(temp_file.name)
  os.rmdir(temp_dir)


#===============================================================================


def _trim_leading_spaces(file_to_read, file_to_write, num_leading_spaces):
  line = file_to_read.readline()
  while line:
    if not line.strip():
      # Write back the empty/whitespace-only line.
      file_to_write.write(line)
      
      # Trim leading spaces from subsequent lines.
      line = file_to_read.readline()
      while line.startswith(" " * num_leading_spaces):
        file_to_write.write(line.lstrip(" "))
        line = file_to_read.readline()
    else:
      file_to_write.write(line)
      line = file_to_read.readline()


def _rename_filenames_inside_file(file_to_read, file_to_write, filenames_to_rename):
  """
  Rename the filenames inside the file that were or will be renamed using the
  `filenames_to_rename` dict.
  """
  
  filenames_to_rename_patterns = {}
  for src_filename, dest_filename in filenames_to_rename.items():
    src_filename_pattern = re.escape(src_filename).replace("\\ ", "\\s")
    filenames_to_rename_patterns[src_filename_pattern] = dest_filename
  
  for line in file_to_read:
    processed_line = line
    for src_filename_pattern, dest_filename in filenames_to_rename_patterns.items():
      processed_line = re.sub(
        src_filename_pattern, lambda match: dest_filename, processed_line)
    file_to_write.write(processed_line)


def _remove_download_link(file_to_read, file_to_write):
  line = file_to_read.readline()
  while line:
    if "download latest release" not in line.lower():
      file_to_write.write(line)
    else:
      break
    line = file_to_read.readline()
  
  # Skip the download link
  line = file_to_read.readline()
  # Skip empty line
  line = file_to_read.readline()
  
  while line:
    file_to_write.write(line)
    line = file_to_read.readline()


FILES_TO_PROCESS = {
  FILENAMES_TO_RENAME["README.md"]: [
    (_trim_leading_spaces, NUM_LEADING_SPACES_TO_TRIM),
    (_rename_filenames_inside_file, FILENAMES_TO_RENAME),
    (_remove_download_link,)
  ]
}

#===============================================================================

{'fraction_non_alphanumeric': 0.1312755333635951, 'fraction_numerical': 0.0034498411257376303, 'mean_word_length': 4.859574468085106, 'pattern_counts': {'":': 6, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3363485', 'n_tokens_mistral': 3675, 'n_tokens_neox': 3559, 'n_words': 809}