<commit_before>/**
 * ISARI Import Scripts People File Definitions
 * =============================================
 */
const moment = require('moment'),
      ENUM_INDEXES = require('../indexes').ENUM_INDEXES,
      helpers = require('../helpers'),
      chalk = require('chalk'),
      partitionBy = helpers.partitionBy,
      hashPeople = helpers.hashPeople,
      overlap = helpers.overlap,
      _ = require('lodash');

module.exports = {
  files: [

    /**
     * SIRH.csv
     * -------------------------------------------------------------------------
     */
    {
      name: 'sirh',
      path: 'people/SIRH.csv',
      delimiter: ',',
      consumer(line, index) {
        const info = {
          year: line.Année,
          name: line['Nom usuel'],
          firstName: line.Prénom,
          birthName: line['Nom de naissance'],
          sirhMatricule: line.Matricule,
          birthDate: line['Date de naissance'],
          gender: line.gender,
          startDate: moment(line['Date de début'], 'YYYY-MM-DD'),
          jobType: line['Type de contrat'],
          gradeSirh: line['Emploi Repère'],
          jobName: line['Emploi Personnalisé'],
          academicMembership: line.Affiliation
        };

        if (line['%ETP'])
          info.timepart = +line['%ETP'].replace(/,/g, '.');

        let nationality = line.Nationalité;

        if (nationality) {
          nationality = ENUM_INDEXES.countries.alpha3[nationality];

          if (!nationality)
            this.error(`Line ${index + 1}: unknown nationality ${chalk.cyan(line.Nationalité)}`);
          else
            info.nationality = nationality.alpha2;
        }

        // Handling endDate
        let endDate;

        if (line['Date fin présumée'] && line['Date de sortie adm'])
          endDate = moment.min(
            moment(line['Date fin présumée'], 'YYYY-MM-DD'),
            moment(line['Date de sortie adm'], 'YYYY-MM-DD')
          );
        else if (line['Date fin présumée'])
          endDate = moment(line['Date fin présumée'], 'YYYY-MM-DD');
        else if (line['Date de sortie adm'])
          endDate = moment(line['Date de sortie adm'], 'YYYY-MM-DD');

        if (endDate)
          info.endDate = endDate;

        return info;
      },
      resolver(lines) {

        // First we need to group the person by matricule
        let persons = partitionBy(lines, 'sirhMatricule');

        // Sorting lines by year
        persons = persons.map(years => {
          return _.sortBy(years, 'year');
        });

        // Creating people objects
        // .filter(p => p[0].name === 'OLIVIER')
        const objects = persons.map(years => {
          const first = years[0],
                last = years[years.length - 1];

          const person = {
            firstName: first.firstName,
            name: last.name,
            sirhMatricule: first.sirhMatricule,
            gender: last.gender,
            nationalities: [last.nationality],
            birthDate: first.birthDate
          };

          if (first.birthName && first.birthName !== first.name)
            person.birthName = first.birthName;

          // Computing positions
          const slices = partitionBy(years, y => `${y.startDate}§${y.endDate || ''}`),
                positions = [];

          slices.forEach((slice, sliceIndex) => {
            const nextSlice = slices[sliceIndex + 1],
                  contract = slice[0],
                  nextContract = (nextSlice || [])[0] || {};

            const position = {
              organization: 'FNSP',
              jobName: contract.jobName,
              jobType: contract.jobType,
              timepart: contract.timepart
            };

            // Dates
            if (nextContract.startDate && contract.endDate) {
              if (nextContract.startDate.isBefore(contract.endDate))
                contract.endDate = nextContract.startDate.subtract(1, 'days');
            }

            if (contract.startDate)
              position.startDate = contract.startDate.format('YYYY-MM-DD');
            if (contract.endDate)
              position.endDate = contract.endDate.format('YYYY-MM-DD');

            // Grades
            position.gradesSirh = _(slice)
              .groupBy('gradeSirh')
              .values()
              .map(grades => grades[0])
              .map((grade, i, grades) => {
                const nextGrade = grades[i + 1];

                const info = {
                  grade: grade.gradeSirh
                };
<commit_msg>SIRH is now jobName null resilient<commit_after>/**
 * ISARI Import Scripts People File Definitions
 * =============================================
 */
const moment = require('moment'),
      ENUM_INDEXES = require('../indexes').ENUM_INDEXES,
      helpers = require('../helpers'),
      chalk = require('chalk'),
      partitionBy = helpers.partitionBy,
      hashPeople = helpers.hashPeople,
      overlap = helpers.overlap,
      _ = require('lodash');

module.exports = {
  files: [

    /**
     * SIRH.csv
     * -------------------------------------------------------------------------
     */
    {
      name: 'sirh',
      path: 'people/SIRH.csv',
      delimiter: ',',
      consumer(line, index) {
        const info = {
          year: line.Année,
          name: line['Nom usuel'],
          firstName: line.Prénom,
          birthName: line['Nom de naissance'],
          sirhMatricule: line.Matricule,
          birthDate: line['Date de naissance'],
          gender: line.gender,
          startDate: moment(line['Date de début'], 'YYYY-MM-DD'),
          jobType: line['Type de contrat'],
          gradeSirh: line['Emploi Repère'],
          jobName: line['Emploi Personnalisé'],
          academicMembership: line.Affiliation
        };

        if (line['%ETP'])
          info.timepart = +line['%ETP'].replace(/,/g, '.');

        let nationality = line.Nationalité;

        if (nationality) {
          nationality = ENUM_INDEXES.countries.alpha3[nationality];

          if (!nationality)
            this.error(`Line ${index + 1}: unknown nationality ${chalk.cyan(line.Nationalité)}`);
          else
            info.nationality = nationality.alpha2;
        }

        // Handling endDate
        let endDate;

        if (line['Date fin présumée'] && line['Date de sortie adm'])
          endDate = moment.min(
            moment(line['Date fin présumée'], 'YYYY-MM-DD'),
            moment(line['Date de sortie adm'], 'YYYY-MM-DD')
          );
        else if (line['Date fin présumée'])
          endDate = moment(line['Date fin présumée'], 'YYYY-MM-DD');
        else if (line['Date de sortie adm'])
          endDate = moment(line['Date de sortie adm'], 'YYYY-MM-DD');

        if (endDate)
          info.endDate = endDate;

        return info;
      },
      resolver(lines) {

        // First we need to group the person by matricule
        let persons = partitionBy(lines, 'sirhMatricule');

        // Sorting lines by year
        persons = persons.map(years => {
          return _.sortBy(years, 'year');
        });

        // Creating people objects
        // .filter(p => p[0].name === 'OLIVIER')
        const objects = persons.map(years => {
          const first = years[0],
                last = years[years.length - 1];

          const person = {
            firstName: first.firstName,
            name: last.name,
            sirhMatricule: first.sirhMatricule,
            gender: last.gender,
            nationalities: [last.nationality],
            birthDate: first.birthDate
          };

          if (first.birthName && first.birthName !== first.name)
            person.birthName = first.birthName;

          // Computing positions
          const slices = partitionBy(years, y => `${y.startDate}§${y.endDate || ''}`),
                positions = [];

          slices.forEach((slice, sliceIndex) => {
            const nextSlice = slices[sliceIndex + 1],
                  contract = slice[0],
                  nextContract = (nextSlice || [])[0] || {};

            const position = {
              organization: 'FNSP',
              jobType: contract.jobType,
              timepart: contract.timepart
            };

            const jobNameLine = slice.find(line => !!line.jobName);

            if (jobNameLine)
              position.jobName = jobNameLine.jobName;

            // Dates
            if (nextContract.startDate && contract.endDate) {
              if (nextContract.startDate.isBefore(contract.endDate))
                contract.endDate = nextContract.startDate.subtract(1, 'days');
            }

            if (contract.startDate)
              position.startDate = contract.startDate.format('YYYY-MM-DD');
            if (contract.endDate)
              position.endDate = contract.endDate.format('YYYY-MM-DD');

            // Grades
            position.gradesSirh = _(slice)
              .groupBy('gradeSirh')
              .values()
              .map(grades => grades[0])
              .map((grade, i, grades) => {
                const nextGrade = grades[i + 1];

                const info = {
                  grade: grade.gradeSirh
                };

{'fraction_non_alphanumeric': 0.1534022863364181, 'fraction_numerical': 0.00261295590636908, 'mean_word_length': 1.6703488372093023, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '642965', 'n_tokens_mistral': 2620, 'n_tokens_neox': 2521, 'n_words': 543}