<commit_before>package org.basex.build;

import static org.basex.core.Text.*;

import java.io.IOException;
import java.util.Locale;
import java.util.regex.Pattern;

import org.basex.core.Prop;
import org.basex.io.IO;
import org.basex.io.IOContent;
import org.basex.io.IOFile;
import org.basex.io.in.BufferInput;
import org.basex.util.TokenBuilder;
import org.basex.util.Util;
import org.basex.util.list.StringList;

/**
 * This class recursively scans files and directories and parses all
 * relevant files.
 *
 * @author BaseX Team 2005-11, BSD License
 * @author <NAME>
 */
public final class DirParser extends TargetParser {
  /** Number of skipped files to log. */
  private static final int SKIPLOG = 10;
  /** Skipped files. */
  private final StringList skipped = new StringList();
  /** File pattern. */
  private final Pattern filter;
  /** Properties. */
  private final Prop prop;
  /** Initial file path. */
  private final String root;
  /** Parse archives in directories. */
  private final boolean archives;
  /** Skip corrupt files in directories. */
  private final boolean skip;

  /** Parser reference. */
  private Parser parser;
  /** Element counter. */
  private int c;

  /**
   * Constructor.
   * @param source source path
   * @param pr database properties
   */
  public DirParser(final IO source, final Prop pr) {
    this(source, "", pr);
  }

  /**
   * Constructor, specifying a target path.
   * @param source source path
   * @param target target path
   * @param pr database properties
   */
  public DirParser(final IO source, final String target, final Prop pr) {
    super(source, target);
    prop = pr;
    final String parent = source.dir();
    root = parent.endsWith("/") ? parent : parent + '/';
    skip = prop.is(Prop.SKIPCORRUPT);
    archives = prop.is(Prop.ADDARCHIVES);
    filter = !source.isDir() ? null :
      Pattern.compile(IOFile.regex(pr.get(Prop.CREATEFILTER)));
  }

  @Override
  public void parse(final Builder build) throws IOException {
    build.meta.filesize = 0;
    build.meta.original = src.path();
    parse(build, src);
  }

  /**
   * Parses the specified file or its children.
   * @param b builder
   * @param io current input
   * @throws IOException I/O exception
   */
  private void parse(final Builder b, final IO io) throws IOException {
    if(io.isDir()) {
      // only {@link IOFile} instances can have children
      for(final IO f : ((IOFile) io).children()) parse(b, f);
    } else {
      src = io;
      if(!archives && src.isArchive()) return;

      // multiple archive files may be parsed in this loop
      while(io.more()) {
        final String nm = Prop.WIN ? io.name().toLowerCase(Locale.ENGLISH) :
          io.name();
        if(filter != null && !filter.matcher(nm).matches()) continue;
        b.meta.filesize += src.length();

        // use global target as prefix
        String targ = trg;
        final String name = src.name();
        String path = src.path();
        // add relative path without root (prefix) and file name (suffix)
        if(path.endsWith('/' + name)) {
          path = path.substring(0, path.length() - name.length());
          if(path.startsWith(root)) path = path.substring(root.length());
          targ = (targ + path).replace("//", "/");
        }

        boolean ok = true;
        IO in = io;
        if(skip) {
          // parse file twice to ensure that it is well-formed
          BufferInput bi = null;
          try {
            // cache file contents to allow or speed up a second run
            bi = io.buffer();
            in = new IOContent(bi.readBytes());
            in.name(io.name());
            parser = Parser.fileParser(in, prop, targ);
            MemBuilder.build("", parser, prop);
          } catch(final IOException ex) {
            Util.debug(ex.getMessage());
            skipped.add(io.path());
            ok = false;
          } finally {
            if(bi != null) bi.close();
          }
        }
        parser = Parser.fileParser(in, prop, targ);

        if(ok) parser.parse(b);
        if(Util.debug && (++c & 0x3FF) == 0) Util.err(";");
      }
    }<commit_msg>[FIX] Storage: SKIPCORRUPT didn't work if last file was corrupt<commit_after>package org.basex.build;

import static org.basex.core.Text.*;

import java.io.IOException;
import java.util.Locale;
import java.util.regex.Pattern;

import org.basex.core.Prop;
import org.basex.io.IO;
import org.basex.io.IOContent;
import org.basex.io.IOFile;
import org.basex.util.TokenBuilder;
import org.basex.util.Util;
import org.basex.util.list.StringList;

/**
 * This class recursively scans files and directories and parses all
 * relevant files.
 *
 * @author BaseX Team 2005-11, BSD License
 * @author <NAME>
 */
public final class DirParser extends TargetParser {
  /** Number of skipped files to log. */
  private static final int SKIPLOG = 10;
  /** Skipped files. */
  private final StringList skipped = new StringList();
  /** File pattern. */
  private final Pattern filter;
  /** Properties. */
  private final Prop prop;
  /** Initial file path. */
  private final String root;
  /** Parse archives in directories. */
  private final boolean archives;
  /** Skip corrupt files in directories. */
  private final boolean skip;

  /** Parser reference. */
  private Parser parser;
  /** Element counter. */
  private int c;

  /**
   * Constructor.
   * @param source source path
   * @param pr database properties
   */
  public DirParser(final IO source, final Prop pr) {
    this(source, "", pr);
  }

  /**
   * Constructor, specifying a target path.
   * @param source source path
   * @param target target path
   * @param pr database properties
   */
  public DirParser(final IO source, final String target, final Prop pr) {
    super(source, target);
    prop = pr;
    final String parent = source.dir();
    root = parent.endsWith("/") ? parent : parent + '/';
    skip = prop.is(Prop.SKIPCORRUPT);
    archives = prop.is(Prop.ADDARCHIVES);
    filter = !source.isDir() ? null :
      Pattern.compile(IOFile.regex(pr.get(Prop.CREATEFILTER)));
  }

  @Override
  public void parse(final Builder build) throws IOException {
    build.meta.filesize = 0;
    build.meta.original = src.path();
    parse(build, src);
  }

  /**
   * Parses the specified file or its children.
   * @param b builder
   * @param io current input
   * @throws IOException I/O exception
   */
  private void parse(final Builder b, final IO io) throws IOException {
    if(io.isDir()) {
      // only {@link IOFile} instances can have children
      for(final IO f : ((IOFile) io).children()) parse(b, f);
    } else {
      src = io;
      if(!archives && src.isArchive()) return;

      // multiple archive files may be parsed in this loop
      while(io.more()) {
        final String nm = Prop.WIN ? io.name().toLowerCase(Locale.ENGLISH) :
          io.name();
        if(filter != null && !filter.matcher(nm).matches()) continue;
        b.meta.filesize += src.length();

        // use global target as prefix
        String targ = trg;
        final String name = src.name();
        String path = src.path();
        // add relative path without root (prefix) and file name (suffix)
        if(path.endsWith('/' + name)) {
          path = path.substring(0, path.length() - name.length());
          if(path.startsWith(root)) path = path.substring(root.length());
          targ = (targ + path).replace("//", "/");
        }

        boolean ok = true;
        IO in = io;
        if(skip) {
          // parse file twice to ensure that it is well-formed
          try {
            // cache file contents to allow or speed up a second run
            in = new IOContent(io.read());
            in.name(io.name());
            parser = Parser.fileParser(in, prop, targ);
            MemBuilder.build("", parser, prop);
          } catch(final IOException ex) {
            Util.debug(ex.getMessage());
            skipped.add(io.path());
            ok = false;
          }
        }

        if(ok) {
          parser = Parser.fileParser(in, prop, targ);
          parser.parse(b);
        } else {
          parser = null;
        }
        if(Util.debug && (++c & 0x3FF) == 0) Util.err(";");
      }
    }
{'fraction_non_alphanumeric': 0.13160131115697463, 'fraction_numerical': 0.0031564890129901664, 'mean_word_length': 2.3789991796554553, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2880500', 'n_tokens_mistral': 2482, 'n_tokens_neox': 2415, 'n_words': 743}