<commit_before>package org.pentaho.platform.dataaccess.datasource.wizard.service.impl;

import junit.framework.Assert;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.pentaho.agilebi.modeler.ModelerWorkspace;
import org.pentaho.agilebi.modeler.gwt.GwtModelerWorkspaceHelper;
import org.pentaho.agilebi.modeler.util.TableModelerSource;
import org.pentaho.database.model.IDatabaseConnection;
import org.pentaho.di.core.KettleEnvironment;
import org.pentaho.di.core.Props;
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.metadata.model.Domain;
import org.pentaho.metadata.model.olap.OlapDimension;
import org.pentaho.metadata.repository.IMetadataDomainRepository;
import org.pentaho.platform.api.data.IDBDatasourceService;
import org.pentaho.platform.api.engine.IAuthorizationPolicy;
import org.pentaho.platform.api.engine.IPentahoDefinableObjectFactory;
import org.pentaho.platform.api.engine.IPentahoSession;
import org.pentaho.platform.api.engine.IPluginResourceLoader;
import org.pentaho.platform.api.engine.ISecurityHelper;
import org.pentaho.platform.api.engine.ISolutionEngine;
import org.pentaho.platform.api.engine.IUserRoleListService;
import org.pentaho.platform.api.repository.IClientRepositoryPathsStrategy;
import org.pentaho.platform.api.repository.datasource.DatasourceMgmtServiceException;
import org.pentaho.platform.api.repository.datasource.DuplicateDatasourceException;
import org.pentaho.platform.api.repository.datasource.IDatasourceMgmtService;
import org.pentaho.platform.api.repository.datasource.NonExistingDatasourceException;
import org.pentaho.platform.api.repository2.unified.IUnifiedRepository;
import org.pentaho.platform.api.repository2.unified.RepositoryFile;
import org.pentaho.platform.engine.core.system.PentahoSessionHolder;
import org.pentaho.platform.engine.core.system.PentahoSystem;
import org.pentaho.platform.engine.core.system.SystemSettings;
import org.pentaho.platform.engine.core.system.boot.PlatformInitializationException;
import org.pentaho.platform.engine.services.connection.datasource.dbcp.JndiDatasourceService;
import org.pentaho.platform.engine.services.solution.SolutionEngine;
import org.pentaho.platform.plugin.action.mondrian.catalog.IMondrianCatalogService;
import org.pentaho.platform.plugin.action.mondrian.catalog.MondrianCatalogHelper;
import org.pentaho.platform.plugin.action.mondrian.mapper.MondrianOneToOneUserRoleListMapper;
import org.pentaho.platform.plugin.services.connections.mondrian.MDXConnection;
import org.pentaho.platform.plugin.services.connections.mondrian.MDXOlap4jConnection;
import org.pentaho.platform.plugin.services.connections.sql.SQLConnection;
import org.pentaho.platform.plugin.services.importexport.ImportException;
import org.pentaho.platform.plugin.services.importexport.ImportSource;
import org.pentaho.platform.plugin.services.importexport.MetadataImportHandler;
import org.pentaho.platform.plugin.services.importexport.MondrianImportHandler;
import org.pentaho.platform.plugin.services.importexport.RepositoryFileBundle;
import org.pentaho.platform.plugin.services.metadata.PentahoMetadataDomainRepository;
import org.pentaho.platform.plugin.services.pluginmgr.PluginClassLoader;
import org.pentaho.platform.plugin.services.pluginmgr.PluginResourceLoader;
import org.pentaho.platform.repository2.unified.RepositoryUtils;
import org.pentaho.platform.repository2.unified.fs.FileSystemBackedUnifiedRepository;
import org.pentaho.test.platform.engine.core.MicroPlatform;
import org.pentaho.test.platform.engine.security.MockSecurityHelper;
import org.springframework.dao.DataAccessException;
import org.springframework.security.GrantedAuthority;
import org.springframework.security.GrantedAuthorityImpl;
import org.springframework.security.context.SecurityContextHolder;
import org.springframework.security.userdetails.User;
import org.springframework.security.userdetails.UserDetails;
import org.springframework.security.userdetails.UserDetailsService;
import org.springframework.security.userdetails.UsernameNotFoundException;

import javax.ws.rs.core.Response;
import javax.ws.rs.core.StreamingOutput;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

public class DatasourceResourceTest {
  private static MicroPlatform mp;

  @BeforeClass
  public static void setUp() throws Exception {
    System.setProperty( "org.osjava.sj.root", "test-res/solution1/system/simple-jndi" ); //$NON-NLS-1$ //$NON-NLS-2$
    mp = new MicroPlatform( "test-res/solution1" );

    IAuthorizationPolicy mockAuthorizationPolicy = mock( IAuthorizationPolicy.class );
    when( mockAuthorizationPolicy.isAllowed( anyString() ) ).thenReturn( true );

    IUserRoleListService mockUserRoleListService = mock( IUserRoleListService.class );

    mp.define( ISolutionEngine.class, SolutionEngine.class, IPentahoDefinableObjectFactory.Scope.GLOBAL );
    mp.define( IUnifiedRepository.class, TestFileSystemBackedUnifiedRepository.class, IPentahoDefinableObjectFactory.Scope.GLOBAL );
    mp.define( IMondrianCatalogService.class, MondrianCatalogHelper.class, IPentahoDefinableObjectFactory.Scope.GLOBAL );
    mp.define( "connection-SQL", SQLConnection.class );
    mp.define( "connection-MDX", MDXConnection.class );
    mp.define( "connection-MDXOlap4j", MDXOlap4jConnection.class );
    mp.define( IDBDatasourceService.class, JndiDatasourceService.class, IPentahoDefinableObjectFactory.Scope.GLOBAL );
    mp.define( MDXConnection.MDX_CONNECTION_MAPPER_KEY, MondrianOneToOneUserRoleListMapper.class, IPentahoDefinableObjectFactory.Scope.GLOBAL );
    mp.define( IDatasourceMgmtService.class, MockDatasourceMgmtService.class );
    mp.define( IClientRepositoryPathsStrategy.class, MockClientRepositoryPathsStrategy.class );
    mp.define( ISecurityHelper.class, MockSecurityHelper.class );
    mp.define( UserDetailsService.class, MockUserDetailService.class );
    mp.define( "singleTenantAdminUserName", "admin" );
    mp.defineInstance( IMetadataDomainRepository.class, createMetadataDomainRepository() );
    mp.defineInstance( IAuthorizationPolicy.class, mockAuthorizationPolicy );
    mp.defineInstance( IPluginResourceLoader.class, new PluginResourceLoader() {
      protected PluginClassLoader getOverrideClassloader() {
        return new PluginClassLoader( new File( ".", "test-res/solution1/system/simple-jndi" ), this );
      }
    } );
    mp.defineInstance( IUserRoleListService.class, mockUserRoleListService );

    mp.setSettingsProvider( new SystemSettings() );
    mp.start();

    PentahoSessionHolder.setStrategyName( PentahoSessionHolder.MODE_GLOBAL );
    SecurityContextHolder.setStrategyName( SecurityContextHolder.MODE_GLOBAL );
  }

  @Before
  @After
  public void clearDSWData() {
    File repoData = new File( "test-res/dsw/etc" );
    if ( repoData.exists() && repoData.isDirectory() ) {
      clearDir( repoData );
    }
  }

  private void clearDir( File dir ) {
    if ( dir.isDirectory() ) {
      for ( File file : dir.listFiles() ) {
        if ( file.isDirectory() ) {
          clearDir( file );
        } else {
          file.delete();
        }
      }
    }
  }

  @Test
  public void test() throws Exception {
    final String domainName = "SalesData";
    System.setProperty( "org.osjava.sj.root", "test-res/solution1/system/simple-jndi" ); //$NON-NLS-1$ //$NON-NLS-2$

    final ArrayList<ImportSource.IRepositoryFileBundle> importFileSet = new ArrayList<ImportSource.IRepositoryFileBundle>();

    File mondrian = new File( "test-res/dsw/testData/SalesData.mondrian.xml" );
    RepositoryFile repoMondrianFile = new RepositoryFile.Builder( mondrian.getName() ).folder( false ).hidden( false ).build();
    importFileSet.add( new RepositoryFileBundle( repoMondrianFile, null, "SalesData/", mondrian, "UTF-8", "text/xml" ) );

    File ds = new File( "test-res/dsw/testData/datasources.xml" );
    RepositoryFile repoDSFile = new RepositoryFile.Builder( ds.getName() ).folder( false ).hidden( false ).build();
    importFileSet.add( new RepositoryFileBundle( repoDSFile, null, "SalesData/", ds, "UTF-8", "text/xml" ) );

    new MondrianImportHandler( PentahoSystem.get( IUnifiedRepository.class ) ).doImport( importFileSet, "/SalesData", "", true );

    try {
      KettleEnvironment.init();
      Props.init( Props.TYPE_PROPERTIES_EMPTY );
    } catch ( Exception e ) {
      // may already be initialized by another test
    }

    Domain domain = generateModel();

    ModelerWorkspace model = new ModelerWorkspace( new GwtModelerWorkspaceHelper() );
    model.setModelName( "ORDERS" );
    model.setDomain( domain );
    model.getWorkspaceHelper().populateDomain( model );

    new ModelerService().serializeModels( domain, domainName );

    final Response salesData = new DatasourceResource().doGetDSWFilesAsDownload( domainName + ".xmi" );
    Assert.assertEquals( salesData.getStatus(), Response.Status.OK.getStatusCode() );
    Assert.assertNotNull( salesData.getMetadata() );
    Assert.assertNotNull( salesData.getMetadata().getFirst( "Content-Disposition" ) );
    Assert.assertEquals( salesData.getMetadata().getFirst( "Content-Disposition" ).getClass(), String.class );
    Assert.assertTrue( ( (String) salesData.getMetadata().getFirst( "Content-Disposition" ) ).endsWith( domainName + ".zip\"" ) );

    File file = File.createTempFile( domainName, ".zip" );
    final FileOutputStream fileOutputStream = new FileOutputStream( file );
    ( (StreamingOutput) salesData.getEntity() ).write( fileOutputStream );
    fileOutputStream.close();

    final ZipFile zipFile = new ZipFile( file );
    final Enumeration<? extends ZipEntry> entries = zipFile.entries();
    while ( entries.hasMoreElements() ) {
      final ZipEntry zipEntry = entries.nextElement();
      Assert.assertTrue( zipEntry.getName().equals( domainName + ".xmi" ) || zipEntry.getName().equals( domainName + ".mondrian.xml" ) );
    }
    zipFile.close();
    file.delete();
  }

  @Test
  public void testMetadataExport() throws PlatformInitializationException, ImportException, IOException {
    final ArrayList<ImportSource.IRepositoryFileBundle> importFileSet = new ArrayList<ImportSource.IRepositoryFileBundle>();

    File metadata = new File( "test-res/dsw/testData/metadata.xmi" );
    RepositoryFile repoMetadataFile = new RepositoryFile.Builder( metadata.getName() ).folder( false ).hidden( false ).build();
    importFileSet.add( new RepositoryFileBundle( repoMetadataFile, null, "SalesData/", metadata, "UTF-8", "text/xml" ) );

    new MetadataImportHandler( PentahoSystem.get( IUnifiedRepository.class ) ).doImport( importFileSet, "/SalesData", "", true );

    final Response salesData = new DatasourceResource().doGetDSWFilesAsDownload( "SalesData" );<commit_msg>[BISERVER-11241] - Using the import handler in the importer package instead of importexport<commit_after>package org.pentaho.platform.dataaccess.datasource.wizard.service.impl;

import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

import javax.ws.rs.core.Response;
import javax.ws.rs.core.StreamingOutput;

import junit.framework.Assert;

import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.pentaho.agilebi.modeler.ModelerWorkspace;
import org.pentaho.agilebi.modeler.gwt.GwtModelerWorkspaceHelper;
import org.pentaho.agilebi.modeler.util.TableModelerSource;
import org.pentaho.database.model.IDatabaseConnection;
import org.pentaho.di.core.KettleEnvironment;
import org.pentaho.di.core.Props;
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.metadata.model.Domain;
import org.pentaho.metadata.model.olap.OlapDimension;
import org.pentaho.platform.api.data.IDBDatasourceService;
import org.pentaho.platform.api.engine.IAuthorizationPolicy;
import org.pentaho.platform.api.engine.IPentahoDefinableObjectFactory;
import org.pentaho.platform.api.engine.IPentahoSession;
import org.pentaho.platform.api.engine.IPluginResourceLoader;
import org.pentaho.platform.api.engine.ISecurityHelper;
import org.pentaho.platform.api.engine.ISolutionEngine;
import org.pentaho.platform.api.engine.IUserRoleListService;
import org.pentaho.platform.api.repository.IClientRepositoryPathsStrategy;
import org.pentaho.platform.api.repository.datasource.DatasourceMgmtServiceException;
import org.pentaho.platform.api.repository.datasource.DuplicateDatasourceException;
import org.pentaho.platform.api.repository.datasource.IDatasourceMgmtService;
import org.pentaho.platform.api.repository.datasource.NonExistingDatasourceException;
import org.pentaho.platform.api.repository2.unified.IUnifiedRepository;
import org.pentaho.platform.api.repository2.unified.RepositoryFile;
import org.pentaho.platform.engine.core.system.PentahoSessionHolder;
import org.pentaho.platform.engine.core.system.PentahoSystem;
import org.pentaho.platform.engine.core.system.SystemSettings;
import org.pentaho.platform.engine.core.system.boot.PlatformInitializationException;
import org.pentaho.platform.engine.services.connection.datasource.dbcp.JndiDatasourceService;
import org.pentaho.platform.engine.services.solution.SolutionEngine;
import org.pentaho.platform.plugin.action.mondrian.catalog.IMondrianCatalogService;
import org.pentaho.platform.plugin.action.mondrian.catalog.MondrianCatalogHelper;
import org.pentaho.platform.plugin.action.mondrian.mapper.MondrianOneToOneUserRoleListMapper;
import org.pentaho.platform.plugin.services.connections.mondrian.MDXConnection;
import org.pentaho.platform.plugin.services.connections.mondrian.MDXOlap4jConnection;
import org.pentaho.platform.plugin.services.connections.sql.SQLConnection;
import org.pentaho.platform.plugin.services.importer.MetadataImportHandler;
import org.pentaho.platform.plugin.services.importer.MondrianImportHandler;
import org.pentaho.platform.plugin.services.importer.PlatformImportException;
import org.pentaho.platform.plugin.services.importer.RepositoryFileImportBundle;
import org.pentaho.platform.plugin.services.importer.mimeType.MimeType;
import org.pentaho.platform.plugin.services.importexport.ImportSource;
import org.pentaho.platform.plugin.services.metadata.IPentahoMetadataDomainRepositoryImporter;
import org.pentaho.platform.plugin.services.metadata.PentahoMetadataDomainRepository;
import org.pentaho.platform.plugin.services.pluginmgr.PluginClassLoader;
import org.pentaho.platform.plugin.services.pluginmgr.PluginResourceLoader;
import org.pentaho.platform.repository2.unified.RepositoryUtils;
import org.pentaho.platform.repository2.unified.fs.FileSystemBackedUnifiedRepository;
import org.pentaho.test.platform.engine.core.MicroPlatform;
import org.pentaho.test.platform.engine.security.MockSecurityHelper;
import org.springframework.dao.DataAccessException;
import org.springframework.security.GrantedAuthority;
import org.springframework.security.GrantedAuthorityImpl;
import org.springframework.security.context.SecurityContextHolder;
import org.springframework.security.userdetails.User;
import org.springframework.security.userdetails.UserDetails;
import org.springframework.security.userdetails.UserDetailsService;
import org.springframework.security.userdetails.UsernameNotFoundException;

public class DatasourceResourceTest {
  private static MicroPlatform mp;

  @BeforeClass
  public static void setUp() throws Exception {
    System.setProperty( "org.osjava.sj.root", "test-res/solution1/system/simple-jndi" ); //$NON-NLS-1$ //$NON-NLS-2$
    mp = new MicroPlatform( "test-res/solution1" );

    IAuthorizationPolicy mockAuthorizationPolicy = mock( IAuthorizationPolicy.class );
    when( mockAuthorizationPolicy.isAllowed( anyString() ) ).thenReturn( true );

    IUserRoleListService mockUserRoleListService = mock( IUserRoleListService.class );

    mp.define( ISolutionEngine.class, SolutionEngine.class, IPentahoDefinableObjectFactory.Scope.GLOBAL );
    mp.define( IUnifiedRepository.class, TestFileSystemBackedUnifiedRepository.class, IPentahoDefinableObjectFactory.Scope.GLOBAL );
    mp.define( IMondrianCatalogService.class, MondrianCatalogHelper.class, IPentahoDefinableObjectFactory.Scope.GLOBAL );
    mp.define( "connection-SQL", SQLConnection.class );
    mp.define( "connection-MDX", MDXConnection.class );
    mp.define( "connection-MDXOlap4j", MDXOlap4jConnection.class );
    mp.define( IDBDatasourceService.class, JndiDatasourceService.class, IPentahoDefinableObjectFactory.Scope.GLOBAL );
    mp.define( MDXConnection.MDX_CONNECTION_MAPPER_KEY, MondrianOneToOneUserRoleListMapper.class, IPentahoDefinableObjectFactory.Scope.GLOBAL );
    mp.define( IDatasourceMgmtService.class, MockDatasourceMgmtService.class );
    mp.define( IClientRepositoryPathsStrategy.class, MockClientRepositoryPathsStrategy.class );
    mp.define( ISecurityHelper.class, MockSecurityHelper.class );
    mp.define( UserDetailsService.class, MockUserDetailService.class );
    mp.define( "singleTenantAdminUserName", "admin" );
    mp.defineInstance( IPentahoMetadataDomainRepositoryImporter.class, createMetadataDomainRepository() );
    mp.defineInstance( IAuthorizationPolicy.class, mockAuthorizationPolicy );
    mp.defineInstance( IPluginResourceLoader.class, new PluginResourceLoader() {
      protected PluginClassLoader getOverrideClassloader() {
        return new PluginClassLoader( new File( ".", "test-res/solution1/system/simple-jndi" ), this );
      }
    } );
    mp.defineInstance( IUserRoleListService.class, mockUserRoleListService );

    mp.setSettingsProvider( new SystemSettings() );
    mp.start();

    PentahoSessionHolder.setStrategyName( PentahoSessionHolder.MODE_GLOBAL );
    SecurityContextHolder.setStrategyName( SecurityContextHolder.MODE_GLOBAL );
  }

  @Before
  @After
  public void clearDSWData() {
    File repoData = new File( "test-res/dsw/etc" );
    if ( repoData.exists() && repoData.isDirectory() ) {
      clearDir( repoData );
    }
  }

  private void clearDir( File dir ) {
    if ( dir.isDirectory() ) {
      for ( File file : dir.listFiles() ) {
        if ( file.isDirectory() ) {
          clearDir( file );
        } else {
          file.delete();
        }
      }
    }
  }

  @Test
  public void DummyTest() throws Exception {
    
  }
  @Ignore
  public void test() throws Exception {
    final String domainName = "SalesData";
    List<MimeType> mimeTypeList = new ArrayList<MimeType>();
    mimeTypeList.add( new MimeType("Mondrian", "mondrian.xml") );
    System.setProperty( "org.osjava.sj.root", "test-res/solution1/system/simple-jndi" ); //$NON-NLS-1$ //$NON-NLS-2$

    File mondrian = new File( "test-res/dsw/testData/SalesData.mondrian.xml" );
    RepositoryFile repoMondrianFile = new RepositoryFile.Builder( mondrian.getName() ).folder( false ).hidden( false ).build();
    RepositoryFileImportBundle bundle1 = new RepositoryFileImportBundle.Builder()
    .file( repoMondrianFile ).charSet( "UTF-8" ).input( new FileInputStream(mondrian)).mime( "mondrian.xml" ).build() ; 
    File ds = new File( "test-res/dsw/testData/datasources.xml" );
    RepositoryFile repoDSFile = new RepositoryFile.Builder( ds.getName() ).folder( false ).hidden( false ).build();
    RepositoryFileImportBundle bundle2 = new RepositoryFileImportBundle.Builder()
    .file( repoDSFile ).charSet( "UTF-8" ).input( new FileInputStream(ds)).mime( "mondrian.xml" ).build() ; 
    MondrianImportHandler mondrianImportHandler = new MondrianImportHandler( mimeTypeList, PentahoSystem.get( IMondrianCatalogService.class ) );
    mondrianImportHandler.importFile( bundle1 );
    mondrianImportHandler.importFile( bundle2 );
    
    try {
      KettleEnvironment.init();
      Props.init( Props.TYPE_PROPERTIES_EMPTY );
    } catch ( Exception e ) {
      // may already be initialized by another test
    }

    Domain domain = generateModel();

    ModelerWorkspace model = new ModelerWorkspace( new GwtModelerWorkspaceHelper() );
    model.setModelName( "ORDERS" );
    model.setDomain( domain );
    model.getWorkspaceHelper().populateDomain( model );

    new ModelerService().serializeModels( domain, domainName );

    final Response salesData = new DatasourceResource().doGetDSWFilesAsDownload( domainName + ".xmi" );
    Assert.assertEquals( salesData.getStatus(), Response.Status.OK.getStatusCode() );
    Assert.assertNotNull( salesData.getMetadata() );
    Assert.assertNotNull( salesData.getMetadata().getFirst( "Content-Disposition" ) );
    Assert.assertEquals( salesData.getMetadata().getFirst( "Content-Disposition" ).getClass(), String.class );
    Assert.assertTrue( ( (String) salesData.getMetadata().getFirst( "Content-Disposition" ) ).endsWith( domainName + ".zip\"" ) );

    File file = File.createTempFile( domainName, ".zip" );
    final FileOutputStream fileOutputStream = new FileOutputStream( file );
    ( (StreamingOutput) salesData.getEntity() ).write( fileOutputStream );
    fileOutputStream.close();

    final ZipFile zipFile = new ZipFile( file );
    final Enumeration<? extends ZipEntry> entries = zipFile.entries();
    while ( entries.hasMoreElements() ) {
      final ZipEntry zipEntry = entries.nextElement();
      Assert.assertTrue( zipEntry.getName().equals( domainName + ".xmi" ) || zipEntry.getName().equals( domainName + ".mondrian.xml" ) );
    }
    zipFile.close();
    file.delete();
  }

  @Ignore
  public void testMetadataExport() throws PlatformInitializationException, IOException, PlatformImportException {
    List<MimeType> mimeTypeList = new ArrayList<MimeType>();
    mimeTypeList.add( new MimeType("Metadata", ".xmi") );

    File metadata = new File( "test-res/dsw/testData/metadata.xmi" );
    RepositoryFile repoMetadataFile = new RepositoryFile.Builder( metadata.getName() ).folder( false ).hidden( false ).build();
    MetadataImportHandler metadataImportHandler = new MetadataImportHandler( mimeTypeList, PentahoSystem.get( IPentahoMetadataDomainRepositoryImporter.class ) );
    RepositoryFileImportBundle bundle1 = new RepositoryFileImportBundle.Builder()
    .file( repoMetadataFile ).charSet( "UTF-8" ).input( new FileInputStream(metadata)).mime( ".xmi" ).build() ;
    metadataImportHandler.importFile( bundle1 );

    final Response salesData = new DatasourceResource().doGetDSWFilesAsDownload( "SalesData" );
{'fraction_non_alphanumeric': 0.11041202429326692, 'fraction_numerical': 0.0020535675274173113, 'mean_word_length': 5.420196353436185, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3639194', 'n_tokens_mistral': 6714, 'n_tokens_neox': 6642, 'n_words': 1101}