<commit_before>/*
 * Copyright (c) 2005-2010 Grameen Foundation USA
 * All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 *
 * See also http://www.apache.org/licenses/LICENSE-2.0.html for an
 * explanation of the license and how it is applied.
 */

package org.mifos.customers.personnel.persistence;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.MissingResourceException;
import java.util.ResourceBundle;
import java.util.Set;

import org.hibernate.Criteria;
import org.hibernate.FetchMode;
import org.hibernate.Hibernate;
import org.hibernate.Session;
import org.hibernate.criterion.LogicalExpression;
import org.hibernate.criterion.Order;
import org.hibernate.criterion.Restrictions;
import org.mifos.accounts.savings.persistence.GenericDao;
import org.mifos.application.NamedQueryConstants;
import org.mifos.application.master.business.CustomFieldDefinitionEntity;
import org.mifos.application.master.util.helpers.MasterConstants;
import org.mifos.application.util.helpers.EntityType;
import org.mifos.customers.personnel.business.PersonnelBO;
import org.mifos.customers.personnel.business.PersonnelRoleEntity;
import org.mifos.customers.personnel.util.helpers.PersonnelLevel;
import org.mifos.customers.personnel.util.helpers.PersonnelStatus;
import org.mifos.dto.domain.CenterCreation;
import org.mifos.dto.domain.PersonnelDto;
import org.mifos.dto.domain.UserDetailDto;
import org.mifos.dto.domain.UserSearchDto;
import org.mifos.dto.screen.SystemUserSearchResultsDto;
import org.mifos.framework.hibernate.helper.StaticHibernateUtil;
import org.mifos.security.MifosUser;
import org.mifos.security.rolesandpermission.business.RoleBO;
import org.springframework.security.core.GrantedAuthority;
import org.springframework.security.core.authority.GrantedAuthorityImpl;

public class PersonnelDaoHibernate implements PersonnelDao {

    private final GenericDao genericDao;

    private static final ResourceBundle activityIdToRolesMap = ResourceBundle.getBundle("org.mifos.security.rolesandpermission.mifos_activity_role");

    public PersonnelDaoHibernate(GenericDao genericDao) {
        this.genericDao = genericDao;
    }

    @Override
    public void save(PersonnelBO user) {
        this.genericDao.createOrUpdate(user);
    }

    @Override
    public PersonnelBO findPersonnelById(Short id) {
        if (id == null) {
            return null;
        }

        HashMap<String, Object> queryParameters = new HashMap<String, Object>();
        queryParameters.put("PERSONNEL_ID", id);

        PersonnelBO personnel = (PersonnelBO) this.genericDao.executeUniqueResultNamedQuery("findPersonnelById", queryParameters);
        if (personnel != null) {
            Hibernate.initialize(personnel.getPreferredLocale().getLanguage());
        }
        return personnel;
    }

    @Override
    public PersonnelBO findPersonnelByUsername(final String username) {
        HashMap<String, Object> queryParameters = new HashMap<String, Object>();
        queryParameters.put("USER_NAME", username);

        return (PersonnelBO) this.genericDao.executeUniqueResultNamedQuery(NamedQueryConstants.GETPERSONNELBYNAME, queryParameters);
    }

    @SuppressWarnings("unchecked")
    @Override
    public MifosUser findAuthenticatedUserByUsername(String username) {
        PersonnelBO user = findPersonnelByUsername(username);
        if (user == null) {
            return null;
        }

        Set<Short> roleIds = new HashSet<Short>();
        for (PersonnelRoleEntity personnelRole : user.getPersonnelRoles()) {
            RoleBO role = personnelRole.getRole();
            roleIds.add(role.getId());
        }

        List<Short> activityIds = new ArrayList<Short>();
        if (!roleIds.isEmpty()) {
            HashMap<String, Object> queryParameters = new HashMap<String, Object>();
            queryParameters.put("ROLE_IDS", new ArrayList<Short>(roleIds));
            activityIds = (List<Short>) this.genericDao.executeNamedQuery("findDistinctActivityIdsForGivenSetOfRoleIds", queryParameters);
        }

        byte[] password = <PASSWORD>();
        boolean enabled = user.isActive();
        boolean accountNonExpired = !user.isLocked();
        boolean credentialsNonExpired = true;
        boolean accountNonLocked = !user.isLocked();

        List<GrantedAuthority> authorities = getGrantedActivityAuthorities(activityIds);

        return new MifosUser(user.getPersonnelId(), user.getOffice().getOfficeId(), user.getLevelEnum().getValue(), new ArrayList<Short>(roleIds), username, password, enabled, accountNonExpired, credentialsNonExpired, accountNonLocked, authorities);
    }

    @SuppressWarnings("unchecked")
    @Override
    public List<PersonnelDto> findActiveLoanOfficersForOffice(CenterCreation centerCreationDto) {

        List<PersonnelDto> activeLoanOfficers = new ArrayList<PersonnelDto>();

        HashMap<String, Object> queryParameters = new HashMap<String, Object>();
        queryParameters.put("levelId", PersonnelLevel.LOAN_OFFICER.getValue());
        queryParameters.put("userId", centerCreationDto.getLoggedInUserId());
        queryParameters.put("userLevelId", centerCreationDto.getLoggedInUserLevelId());
        queryParameters.put("officeId", centerCreationDto.getOfficeId());
        queryParameters.put("statusId", PersonnelStatus.ACTIVE.getValue());

        List<PersonnelDto> queryResult = (List<PersonnelDto>) genericDao.executeNamedQuery(
                NamedQueryConstants.MASTERDATA_ACTIVE_LOANOFFICERS_INBRANCH, queryParameters);
        if (queryResult != null) {
            activeLoanOfficers.addAll(queryResult);
        }

        return activeLoanOfficers;
    }

    @Override
    public PersonnelBO findByGlobalPersonnelNum(String globalNumber) {

        HashMap<String, Object> queryParameters = new HashMap<String, Object>();
        queryParameters.put("globalPersonnelNum", globalNumber);

        PersonnelBO personnel = (PersonnelBO) this.genericDao.executeUniqueResultNamedQuery(NamedQueryConstants.PERSONNEL_BY_SYSTEM_ID, queryParameters);
        if (personnel != null) {
            Hibernate.initialize(personnel.getPreferredLocale().getLanguage());
        }
        return personnel;
    }

    private List<GrantedAuthority> getGrantedActivityAuthorities(List<Short> activityIds) {
        List<GrantedAuthority> authorities = new ArrayList<GrantedAuthority>();
        for(Short activityId : activityIds) {
            authorities.add(new GrantedAuthorityImpl(getRoleForActivityId(activityId.toString())));
        }
        return authorities;
    }

    private String getRoleForActivityId(String activityId) {
        try {
            return activityIdToRolesMap.getString(activityId).trim();
        } catch (MissingResourceException e) {
            return "ROLE_UNDEFINED";
        }
    }

    @SuppressWarnings("unchecked")
    @Override
    public final List<CustomFieldDefinitionEntity> retrieveCustomFieldEntitiesForPersonnel() {
        Map<String, Object> queryParameters = new HashMap<String, Object>();
        queryParameters.put(MasterConstants.ENTITY_TYPE, EntityType.PERSONNEL.getValue());
        return (List<CustomFieldDefinitionEntity>) genericDao.executeNamedQuery(NamedQueryConstants.RETRIEVE_CUSTOM_FIELDS, queryParameters);
    }

    @SuppressWarnings("unchecked")
    @Override
    public List<Object[]> getCustomFieldResponses(List<Short> customFieldIds) {
        Map<String, Object> queryParameters = new HashMap<String, Object>();
        queryParameters.put("CUSTOM_FIELD_ID", customFieldIds);
        return (List<Object[]>) genericDao.executeNamedQuery("PersonnelCustomFieldEntity.getResponses", queryParameters);
    }

    @SuppressWarnings("unchecked")
    @Override
    public SystemUserSearchResultsDto search(UserSearchDto searchDto, MifosUser user) {

        Short userId = Integer.valueOf(user.getUserId()).shortValue();

        PersonnelBO loggedInUser = findPersonnelById(userId);

        final PersonnelLevel level = loggedInUser.getLevelEnum();
        final String searchAllSubOfficesInclusiveOfLoggedInUserOffice = loggedInUser.getOfficeSearchId() + "%";
        final String searchString = org.mifos.framework.util.helpers.SearchUtils.normalizeSearchString(searchDto.getSearchTerm());
        final String username = searchString + "%";
        String firstName = "";
        String secondPartOfName = "";

        HashMap<String, Object> queryParameters = new HashMap<String, Object>();
        queryParameters.put("SEARCH_ALL", searchAllSubOfficesInclusiveOfLoggedInUserOffice);
        queryParameters.put("USERID", userId);
        queryParameters.put("LOID", PersonnelLevel.LOAN_OFFICER.getValue());
        queryParameters.put("USERLEVEL_ID", level.getValue());
        queryParameters.put("USER_NAME", username);

        if (searchString.contains(" ")) {
            firstName = searchString.substring(0, searchString.indexOf(" "));
            secondPartOfName = searchString.substring(searchString.indexOf(" ") + 1,searchString.length());
            queryParameters.put("USER_NAME1", firstName);
            queryParameters.put("USER_NAME2", secondPartOfName);
        } else {
            firstName = searchString;
            secondPartOfName = "";
            queryParameters.put("USER_NAME1", searchString);
            queryParameters.put("USER_NAME2", "");
        }

        Long searchResultsCount = (Long) this.genericDao.executeUniqueResultNamedQuery(NamedQueryConstants.PERSONNEL_SEARCH_COUNT, queryParameters);

        Session session = StaticHibernateUtil.getSessionTL();
        Criteria criteriaQuery = session.createCriteria(PersonnelBO.class);
        criteriaQuery.createAlias("office", "o");
        criteriaQuery.createAlias("personnelDetails", "d");

        if (PersonnelLevel.LOAN_OFFICER.getValue().equals(Short.valueOf("2"))) {
            criteriaQuery.add(Restrictions.eq("personnelId", userId));
        }
        criteriaQuery.add(Restrictions.like("o.searchId", searchAllSubOfficesInclusiveOfLoggedInUserOffice));

        LogicalExpression firstOrLastNameMatchUsername =  Restrictions.or(Restrictions.like("d.name.firstName", username), Restrictions.like("d.name.lastName", username));
        LogicalExpression firstNameAndLastNameMatchGivenParts = Restrictions.and(Restrictions.like("d.name.firstName", firstName), Restrictions.like("d.name.lastName", secondPartOfName));

        criteriaQuery.add(Restrictions.or(firstOrLastNameMatchUsername, firstNameAndLastNameMatchGivenParts));
        criteriaQuery.addOrder(Order.asc("o.officeName"));
        criteriaQuery.addOrder(Order.asc("d.name.lastName"));

        criteriaQuery.setFetchMode("office", FetchMode.JOIN);
        criteriaQuery.setFetchMode("level", FetchMode.JOIN);
        criteriaQuery.setFetchMode("personnelDetails", FetchMode.JOIN);

        int firstResult = (searchDto.getPage() * searchDto.getPageSize()) - searchDto.getPageSize();
        criteriaQuery.setFirstResult(firstResult);
        criteriaQuery.setMaxResults(searchDto.getPageSize());

        List<PersonnelBO> pagedResults = criteriaQuery.list();
        List<UserDetailDto> pagedUserDetails = new ArrayList<UserDetailDto>();
        for (PersonnelBO personnelBO : pagedResults) {
            pagedUserDetails.add(personnelBO.toDto());
        }

        SystemUserSearchResultsDto resultsDto = new SystemUserSearchResultsDto(searchResultsCount.intValue(), firstResult, searchDto.getPage(), searchDto.getPageSize(), pagedUserDetails);

        return resultsDto;
    }
}<commit_msg>MIFOS-4385: fix initialize language supported entity when fetching personnel details<commit_after>/*
 * Copyright (c) 2005-2010 Grameen Foundation USA
 * All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 *
 * See also http://www.apache.org/licenses/LICENSE-2.0.html for an
 * explanation of the license and how it is applied.
 */

package org.mifos.customers.personnel.persistence;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.MissingResourceException;
import java.util.ResourceBundle;
import java.util.Set;

import org.hibernate.Criteria;
import org.hibernate.FetchMode;
import org.hibernate.Hibernate;
import org.hibernate.Session;
import org.hibernate.criterion.LogicalExpression;
import org.hibernate.criterion.Order;
import org.hibernate.criterion.Restrictions;
import org.mifos.accounts.savings.persistence.GenericDao;
import org.mifos.application.NamedQueryConstants;
import org.mifos.application.master.business.CustomFieldDefinitionEntity;
import org.mifos.application.master.util.helpers.MasterConstants;
import org.mifos.application.util.helpers.EntityType;
import org.mifos.customers.personnel.business.PersonnelBO;
import org.mifos.customers.personnel.business.PersonnelRoleEntity;
import org.mifos.customers.personnel.util.helpers.PersonnelLevel;
import org.mifos.customers.personnel.util.helpers.PersonnelStatus;
import org.mifos.dto.domain.CenterCreation;
import org.mifos.dto.domain.PersonnelDto;
import org.mifos.dto.domain.UserDetailDto;
import org.mifos.dto.domain.UserSearchDto;
import org.mifos.dto.screen.SystemUserSearchResultsDto;
import org.mifos.framework.hibernate.helper.StaticHibernateUtil;
import org.mifos.security.MifosUser;
import org.mifos.security.rolesandpermission.business.RoleBO;
import org.springframework.security.core.GrantedAuthority;
import org.springframework.security.core.authority.GrantedAuthorityImpl;

public class PersonnelDaoHibernate implements PersonnelDao {

    private final GenericDao genericDao;

    private static final ResourceBundle activityIdToRolesMap = ResourceBundle.getBundle("org.mifos.security.rolesandpermission.mifos_activity_role");

    public PersonnelDaoHibernate(GenericDao genericDao) {
        this.genericDao = genericDao;
    }

    @Override
    public void save(PersonnelBO user) {
        this.genericDao.createOrUpdate(user);
    }

    @Override
    public PersonnelBO findPersonnelById(Short id) {
        if (id == null) {
            return null;
        }

        HashMap<String, Object> queryParameters = new HashMap<String, Object>();
        queryParameters.put("PERSONNEL_ID", id);

        PersonnelBO personnel = (PersonnelBO) this.genericDao.executeUniqueResultNamedQuery("findPersonnelById", queryParameters);
        if (personnel != null) {
            Hibernate.initialize(personnel.getPreferredLocale());
            if (personnel.getPreferredLocale() != null) {
                Hibernate.initialize(personnel.getPreferredLocale().getLanguage());
            }
        }
        return personnel;
    }

    @Override
    public PersonnelBO findPersonnelByUsername(final String username) {
        HashMap<String, Object> queryParameters = new HashMap<String, Object>();
        queryParameters.put("USER_NAME", username);

        return (PersonnelBO) this.genericDao.executeUniqueResultNamedQuery(NamedQueryConstants.GETPERSONNELBYNAME, queryParameters);
    }

    @SuppressWarnings("unchecked")
    @Override
    public MifosUser findAuthenticatedUserByUsername(String username) {
        PersonnelBO user = findPersonnelByUsername(username);
        if (user == null) {
            return null;
        }

        Set<Short> roleIds = new HashSet<Short>();
        for (PersonnelRoleEntity personnelRole : user.getPersonnelRoles()) {
            RoleBO role = personnelRole.getRole();
            roleIds.add(role.getId());
        }

        List<Short> activityIds = new ArrayList<Short>();
        if (!roleIds.isEmpty()) {
            HashMap<String, Object> queryParameters = new HashMap<String, Object>();
            queryParameters.put("ROLE_IDS", new ArrayList<Short>(roleIds));
            activityIds = (List<Short>) this.genericDao.executeNamedQuery("findDistinctActivityIdsForGivenSetOfRoleIds", queryParameters);
        }

        byte[] password = <PASSWORD>();
        boolean enabled = user.isActive();
        boolean accountNonExpired = !user.isLocked();
        boolean credentialsNonExpired = true;
        boolean accountNonLocked = !user.isLocked();

        List<GrantedAuthority> authorities = getGrantedActivityAuthorities(activityIds);

        return new MifosUser(user.getPersonnelId(), user.getOffice().getOfficeId(), user.getLevelEnum().getValue(), new ArrayList<Short>(roleIds), username, password, enabled, accountNonExpired, credentialsNonExpired, accountNonLocked, authorities);
    }

    @SuppressWarnings("unchecked")
    @Override
    public List<PersonnelDto> findActiveLoanOfficersForOffice(CenterCreation centerCreationDto) {

        List<PersonnelDto> activeLoanOfficers = new ArrayList<PersonnelDto>();

        HashMap<String, Object> queryParameters = new HashMap<String, Object>();
        queryParameters.put("levelId", PersonnelLevel.LOAN_OFFICER.getValue());
        queryParameters.put("userId", centerCreationDto.getLoggedInUserId());
        queryParameters.put("userLevelId", centerCreationDto.getLoggedInUserLevelId());
        queryParameters.put("officeId", centerCreationDto.getOfficeId());
        queryParameters.put("statusId", PersonnelStatus.ACTIVE.getValue());

        List<PersonnelDto> queryResult = (List<PersonnelDto>) genericDao.executeNamedQuery(
                NamedQueryConstants.MASTERDATA_ACTIVE_LOANOFFICERS_INBRANCH, queryParameters);
        if (queryResult != null) {
            activeLoanOfficers.addAll(queryResult);
        }

        return activeLoanOfficers;
    }

    @Override
    public PersonnelBO findByGlobalPersonnelNum(String globalNumber) {

        HashMap<String, Object> queryParameters = new HashMap<String, Object>();
        queryParameters.put("globalPersonnelNum", globalNumber);

        PersonnelBO personnel = (PersonnelBO) this.genericDao.executeUniqueResultNamedQuery(NamedQueryConstants.PERSONNEL_BY_SYSTEM_ID, queryParameters);
        if (personnel != null) {
            Hibernate.initialize(personnel.getPreferredLocale().getLanguage());
        }
        return personnel;
    }

    private List<GrantedAuthority> getGrantedActivityAuthorities(List<Short> activityIds) {
        List<GrantedAuthority> authorities = new ArrayList<GrantedAuthority>();
        for(Short activityId : activityIds) {
            authorities.add(new GrantedAuthorityImpl(getRoleForActivityId(activityId.toString())));
        }
        return authorities;
    }

    private String getRoleForActivityId(String activityId) {
        try {
            return activityIdToRolesMap.getString(activityId).trim();
        } catch (MissingResourceException e) {
            return "ROLE_UNDEFINED";
        }
    }

    @SuppressWarnings("unchecked")
    @Override
    public final List<CustomFieldDefinitionEntity> retrieveCustomFieldEntitiesForPersonnel() {
        Map<String, Object> queryParameters = new HashMap<String, Object>();
        queryParameters.put(MasterConstants.ENTITY_TYPE, EntityType.PERSONNEL.getValue());
        return (List<CustomFieldDefinitionEntity>) genericDao.executeNamedQuery(NamedQueryConstants.RETRIEVE_CUSTOM_FIELDS, queryParameters);
    }

    @SuppressWarnings("unchecked")
    @Override
    public List<Object[]> getCustomFieldResponses(List<Short> customFieldIds) {
        Map<String, Object> queryParameters = new HashMap<String, Object>();
        queryParameters.put("CUSTOM_FIELD_ID", customFieldIds);
        return (List<Object[]>) genericDao.executeNamedQuery("PersonnelCustomFieldEntity.getResponses", queryParameters);
    }

    @SuppressWarnings("unchecked")
    @Override
    public SystemUserSearchResultsDto search(UserSearchDto searchDto, MifosUser user) {

        Short userId = Integer.valueOf(user.getUserId()).shortValue();

        PersonnelBO loggedInUser = findPersonnelById(userId);

        final PersonnelLevel level = loggedInUser.getLevelEnum();
        final String searchAllSubOfficesInclusiveOfLoggedInUserOffice = loggedInUser.getOfficeSearchId() + "%";
        final String searchString = org.mifos.framework.util.helpers.SearchUtils.normalizeSearchString(searchDto.getSearchTerm());
        final String username = searchString + "%";
        String firstName = "";
        String secondPartOfName = "";

        HashMap<String, Object> queryParameters = new HashMap<String, Object>();
        queryParameters.put("SEARCH_ALL", searchAllSubOfficesInclusiveOfLoggedInUserOffice);
        queryParameters.put("USERID", userId);
        queryParameters.put("LOID", PersonnelLevel.LOAN_OFFICER.getValue());
        queryParameters.put("USERLEVEL_ID", level.getValue());
        queryParameters.put("USER_NAME", username);

        if (searchString.contains(" ")) {
            firstName = searchString.substring(0, searchString.indexOf(" "));
            secondPartOfName = searchString.substring(searchString.indexOf(" ") + 1,searchString.length());
            queryParameters.put("USER_NAME1", firstName);
            queryParameters.put("USER_NAME2", secondPartOfName);
        } else {
            firstName = searchString;
            secondPartOfName = "";
            queryParameters.put("USER_NAME1", searchString);
            queryParameters.put("USER_NAME2", "");
        }

        Long searchResultsCount = (Long) this.genericDao.executeUniqueResultNamedQuery(NamedQueryConstants.PERSONNEL_SEARCH_COUNT, queryParameters);

        Session session = StaticHibernateUtil.getSessionTL();
        Criteria criteriaQuery = session.createCriteria(PersonnelBO.class);
        criteriaQuery.createAlias("office", "o");
        criteriaQuery.createAlias("personnelDetails", "d");

        if (PersonnelLevel.LOAN_OFFICER.getValue().equals(Short.valueOf("2"))) {
            criteriaQuery.add(Restrictions.eq("personnelId", userId));
        }
        criteriaQuery.add(Restrictions.like("o.searchId", searchAllSubOfficesInclusiveOfLoggedInUserOffice));

        LogicalExpression firstOrLastNameMatchUsername =  Restrictions.or(Restrictions.like("d.name.firstName", username), Restrictions.like("d.name.lastName", username));
        LogicalExpression firstNameAndLastNameMatchGivenParts = Restrictions.and(Restrictions.like("d.name.firstName", firstName), Restrictions.like("d.name.lastName", secondPartOfName));

        criteriaQuery.add(Restrictions.or(firstOrLastNameMatchUsername, firstNameAndLastNameMatchGivenParts));
        criteriaQuery.addOrder(Order.asc("o.officeName"));
        criteriaQuery.addOrder(Order.asc("d.name.lastName"));

        criteriaQuery.setFetchMode("office", FetchMode.JOIN);
        criteriaQuery.setFetchMode("level", FetchMode.JOIN);
        criteriaQuery.setFetchMode("personnelDetails", FetchMode.JOIN);

        int firstResult = (searchDto.getPage() * searchDto.getPageSize()) - searchDto.getPageSize();
        criteriaQuery.setFirstResult(firstResult);
        criteriaQuery.setMaxResults(searchDto.getPageSize());

        List<PersonnelBO> pagedResults = criteriaQuery.list();
        List<UserDetailDto> pagedUserDetails = new ArrayList<UserDetailDto>();
        for (PersonnelBO personnelBO : pagedResults) {
            pagedUserDetails.add(personnelBO.toDto());
        }

        SystemUserSearchResultsDto resultsDto = new SystemUserSearchResultsDto(searchResultsCount.intValue(), firstResult, searchDto.getPage(), searchDto.getPageSize(), pagedUserDetails);

        return resultsDto;
    }
}
{'fraction_non_alphanumeric': 0.10193996324280172, 'fraction_numerical': 0.0018787012456606086, 'mean_word_length': 4.1065693430656935, 'pattern_counts': {'":': 0, '<': 87, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3145934', 'n_tokens_mistral': 6697, 'n_tokens_neox': 6693, 'n_words': 1189}