<commit_before>var uploadDir = process.env.PWD + '/.uploads/';

Meteor.startup(function () {
  log.info('logging is ready');

  UploadServer.init({
    tmpDir: uploadDir + 'tmp',
    uploadDir: uploadDir,
    checkCreateDirectories: true //create the directories for you
  });
});

Meteor.methods({
  importIntegradas: function(path) {
    if (Houston._user_is_admin(Meteor.userId())) {
      var resourceId;
      var lines = [];

      CSV.readCsvFileLineByLine(uploadDir + path, {
        headers: true
      }, function(line) {
        lines.push(line);
      });

      schoolNames(lines[0]).forEach(function(schoolDirty) {
        var school = schoolDirty.trim();
        log.info('write school: ' + school);
        if (!Schools.findOne({name: school})) {
          Schools.insert({
            name: school
          }, function(error) {
            if (error) {
              throw new Meteor.Error('Collection-Error', 'Schools: ' + error.message);
            }
          });
        }
      });
      lines.forEach(function(line, index) {
        if (!(line.Title && line.Reference && line.Editor && line.Subject && line.Group)) {
          var lineNumber = index + 2
          console.log('error');
          console.log(line);
          console.log(lineNumber);
          throw new Meteor.Error('Collection-Error', 'There is an error on line ' + lineNumber);
        }
        var title = line.Title.trim();
        var reference = line.Reference.trim();
        var editor = line.Editor.trim();
        var subject = line.Subject.trim();
        var group = line.Group.trim();

        log.info('processing: ' + reference);
        var resource = Resources.findOne({reference: reference});
        if (!resource) {
          resourceId = Resources.insert({
            title:     title,
            reference: reference,
            editor:    editor,
            subject:   subject,
            group:     group
          }, function(error) {
            if (error) {
              throw new Meteor.Error('Collection-Error', 'Reference: '  + reference + ' - ' + error.message);
            }
          });
        } else {
          resourceId = resource._id;
        }

        schoolNames(lines[0]).forEach(function(schoolNameDirty) {
          var schoolName = schoolNameDirty.trim();
          var year = line.Year.trim();
          var courseName = schoolName + '-' + year;
          var school = Schools.findOne({name: schoolName});

          log.info('school: ' + schoolName);
          if (school) {
            log.info('schoolId: ' + school._id);
            if ( line[schoolName] === 'x' ) {
              Courses.upsert({
                name: courseName
              }, {
                $setOnInsert: {
                  name: courseName,
                  year: year,
                  schoolId: school._id,
                  resources: [resourceId]
                }, 
                $addToSet: {
                  resources: resourceId
                }
              }, function(error) {
                if (error) {
                  throw new Meteor.Error('Collection-Error', 'Reference: '  + reference + ' - ' + error.message);
                }
              });
            } else {
              Courses.update({
                name: courseName
              }, {
                $pull: {
                  resources: resourceId
                }
              }, function(error) {
                if (error) {
                  throw new Meteor.Error('Collection-Error', 'Reference: '  + reference + ' - ' + error.message);
                }
              });
            }
          } else {
            throw new Meteor.Error('shcool-not-found', 'No shccol saved with name: ' + schoolName + ' - Reference: ' + Resources.findOne(resourceId).reference);
          }
        });
      });
      UploadServer.delete(path);
    }
  }
});
<commit_msg>new integradas import modifies name and so on<commit_after>var uploadDir = process.env.PWD + '/.uploads/';

Meteor.startup(function () {
  log.info('logging is ready');

  UploadServer.init({
    tmpDir: uploadDir + 'tmp',
    uploadDir: uploadDir,
    checkCreateDirectories: true //create the directories for you
  });
});

Meteor.methods({
  importIntegradas: function(path) {
    if (Houston._user_is_admin(Meteor.userId())) {
      var resourceId;
      var lines = [];

      CSV.readCsvFileLineByLine(uploadDir + path, {
        headers: true
      }, function(line) {
        lines.push(line);
      });

      schoolNames(lines[0]).forEach(function(schoolDirty) {
        var school = schoolDirty.trim();
        log.info('write school: ' + school);
        if (!Schools.findOne({name: school})) {
          Schools.insert({
            name: school
          }, function(error) {
            if (error) {
              throw new Meteor.Error('Collection-Error', 'Schools: ' + error.message);
            }
          });
        }
      });
      lines.forEach(function(line, index) {
        if (!(line.Title && line.Reference && line.Editor && line.Subject && line.Group)) {
          var lineNumber = index + 2
          console.log('error');
          console.log(line);
          console.log(lineNumber);
          throw new Meteor.Error('Collection-Error', 'There is an error on line ' + lineNumber);
        }
        var title = line.Title.trim();
        var reference = line.Reference.trim();
        var editor = line.Editor.trim();
        var subject = line.Subject.trim();
        var group = line.Group.trim();

        log.info('processing: ' + reference);
        var resource = Resources.findOne({reference: reference});
        if (!resource) {
          resourceId = Resources.insert({
            title:     title,
            reference: reference,
            editor:    editor,
            subject:   subject,
            group:     group
          }, function(error) {
            if (error) {
              throw new Meteor.Error('Collection-Error', 'Reference: '  + reference + ' - ' + error.message);
            }
          });
        } else {
          Resources.update({
            _id: resource._id
          }, {
            $set: {
              title:     title,
              editor:    editor,
              subject:   subject,
              group:     group
            }
          })
          resourceId = resource._id;
        }

        schoolNames(lines[0]).forEach(function(schoolNameDirty) {
          var schoolName = schoolNameDirty.trim();
          var year = line.Year.trim();
          var courseName = schoolName + '-' + year;
          var school = Schools.findOne({name: schoolName});

          log.info('school: ' + schoolName);
          if (school) {
            log.info('schoolId: ' + school._id);
            if ( line[schoolName] === 'x' ) {
              Courses.upsert({
                name: courseName
              }, {
                $setOnInsert: {
                  name: courseName,
                  year: year,
                  schoolId: school._id,
                  resources: [resourceId]
                }, 
                $addToSet: {
                  resources: resourceId
                }
              }, function(error) {
                if (error) {
                  throw new Meteor.Error('Collection-Error', 'Reference: '  + reference + ' - ' + error.message);
                }
              });
            } else {
              Courses.update({
                name: courseName
              }, {
                $pull: {
                  resources: resourceId
                }
              }, function(error) {
                if (error) {
                  throw new Meteor.Error('Collection-Error', 'Reference: '  + reference + ' - ' + error.message);
                }
              });
            }
          } else {
            throw new Meteor.Error('shcool-not-found', 'No shccol saved with name: ' + schoolName + ' - Reference: ' + Resources.findOne(resourceId).reference);
          }
        });
      });
      UploadServer.delete(path);
    }
  }
});

{'fraction_non_alphanumeric': 0.12833230006199628, 'fraction_numerical': 0.0007439553626782394, 'mean_word_length': 1.4696876913655847, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '343778', 'n_tokens_mistral': 2179, 'n_tokens_neox': 2096, 'n_words': 431}