<commit_before>/*
 * Copyright (c) 2014-2015, Arjuna Technologies Limited, Newcastle-upon-Tyne, England. All rights reserved.
 */

package com.arjuna.dbutils.metadata.xssf.generator;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.net.URI;
import java.util.LinkedList;
import java.util.List;
import java.util.UUID;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.poi.xssf.usermodel.XSSFCell;
import org.apache.poi.xssf.usermodel.XSSFComment;
import org.apache.poi.xssf.usermodel.XSSFRow;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;

public class XSSFSpreadsheetMetadataGenerator
{
    private static final Logger logger = Logger.getLogger(XSSFSpreadsheetMetadataGenerator.class.getName());

    public String generateXSSFSpeadsheetMetadata(URI baseRDFURI, byte[] spreadsheetData)
    {
        logger.log(Level.FINE, "Generate XSSF Speadsheet Metadata (Data)");

        try
        {
            InputStream xssfWorkbookInputStream = new ByteArrayInputStream(spreadsheetData);
            String      metadata                = generateXSSFSpeadsheetMetadata(baseRDFURI, xssfWorkbookInputStream);
            xssfWorkbookInputStream.close();

            return metadata;
        }
        catch (Throwable throwable)
        {
            logger.log(Level.WARNING, "Problem Generating during XSSF Speadsheet Metadata Scan (Data)", throwable);

            return null;
        }
    }

    public String generateXSSFSpeadsheetMetadata(URI baseRDFURI, File spreadsheetFile)
    {
        logger.log(Level.FINE, "Generate XSSF Speadsheet Metadata (File)");

        try
        {
            InputStream xssfWorkbookInputStream = new FileInputStream(spreadsheetFile);
            String      metadata                = generateXSSFSpeadsheetMetadata(baseRDFURI, xssfWorkbookInputStream);
            xssfWorkbookInputStream.close();

            return metadata;
        }
        catch (Throwable throwable)
        {
            logger.log(Level.WARNING, "Problem Generating during XSSF Speadsheet Metadata Scan (File)", throwable);

            return null;
        }
    }

    private String generateXSSFSpeadsheetMetadata(URI baseRDFURI, InputStream xssfWorkbookInputStream)
    {
        logger.log(Level.FINE, "Generate XSSF Speadsheet Metadata");

        try
        {
            XSSFWorkbook xssfWorkbook = new XSSFWorkbook(xssfWorkbookInputStream);

            StringBuffer rdfText = new StringBuffer();
            rdfText.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n\n");
            rdfText.append("<rdf:RDF xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\" xmlns:rdfs=\"http://www.w3.org/2000/01/rdf-schema#\" xmlns:x=\"http://rdfs.arjuna.com/xssf#\" xmlns:d=\"http://rdfs.arjuna.com/description#\">\n");

            boolean firstItem = true;

            List<String> sheetIds = new LinkedList<String>();
            for (int sheetIndex = 0; sheetIndex < xssfWorkbook.getNumberOfSheets(); sheetIndex++)
            {
                String sheetId = generateXSSFSheetMetadata(rdfText, firstItem, baseRDFURI, xssfWorkbook.getSheetAt(sheetIndex));
                if (sheetId != null)
                    sheetIds.add(sheetId);
                firstItem = firstItem && sheetIds.isEmpty();
            }

            String workbookId = UUID.randomUUID().toString();
            if (! firstItem)
                rdfText.append('\n');
            else
                rdfText.append('\n');
            rdfText.append("    <x:Workbook rdf:about=\"");
            rdfText.append(baseRDFURI.resolve('#' + workbookId));
            rdfText.append("\">\n");
            for (String sheetId: sheetIds)
            {
                rdfText.append("        <x:hasSheet rdf:resource=\"");
                rdfText.append(baseRDFURI.resolve('#' + sheetId.toString()));
                rdfText.append("\"/>\n");
            }
            rdfText.append("    </x:Workbook>\n");

            rdfText.append("</rdf:RDF>\n");

            if (logger.isLoggable(Level.FINEST))
                logger.log(Level.FINEST, "XSSF Speadsheet RDF:\n[\n" + rdfText.toString() + "]");

            return rdfText.toString();
        }
        catch (Throwable throwable)
        {
            logger.log(Level.WARNING, "Problem Generating during XSSF Speadsheet Metadata Scan", throwable);
<commit_msg>Initital support for 'location' and 'filename'<commit_after>/*
 * Copyright (c) 2014-2015, Arjuna Technologies Limited, Newcastle-upon-Tyne, England. All rights reserved.
 */

package com.arjuna.dbutils.metadata.xssf.generator;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.net.URI;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.apache.poi.xssf.usermodel.XSSFCell;
import org.apache.poi.xssf.usermodel.XSSFComment;
import org.apache.poi.xssf.usermodel.XSSFRow;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;

public class XSSFSpreadsheetMetadataGenerator
{
    private static final Logger logger = Logger.getLogger(XSSFSpreadsheetMetadataGenerator.class.getName());

    public String generateXSSFSpeadsheetMetadata(URI baseRDFURI, byte[] spreadsheetData)
    {
        logger.log(Level.FINE, "Generate XSSF Speadsheet Metadata (Data)");

        try
        {
            InputStream xssfWorkbookInputStream = new ByteArrayInputStream(spreadsheetData);
            String      metadata                = generateXSSFSpeadsheetMetadata(baseRDFURI, xssfWorkbookInputStream, null, null);
            xssfWorkbookInputStream.close();

            return metadata;
        }
        catch (Throwable throwable)
        {
            logger.log(Level.WARNING, "Problem Generating during XSSF Speadsheet Metadata Scan (Data)", throwable);

            return null;
        }
    }

    public String generateXSSFSpeadsheetMetadata(URI baseRDFURI, File spreadsheetFile)
    {
        logger.log(Level.FINE, "Generate XSSF Speadsheet Metadata (File)");

        try
        {
            InputStream xssfWorkbookInputStream = new FileInputStream(spreadsheetFile);
            String      metadata                = generateXSSFSpeadsheetMetadata(baseRDFURI, xssfWorkbookInputStream, null, null);
            xssfWorkbookInputStream.close();

            return metadata;
        }
        catch (Throwable throwable)
        {
            logger.log(Level.WARNING, "Problem Generating during XSSF Speadsheet Metadata Scan (File)", throwable);

            return null;
        }
    }

    public String generateXSSFSpeadsheetMetadata(URI baseRDFURI, Map<String, Object> spreadsheetMap)
    {
        logger.log(Level.FINE, "Generate XSSF Speadsheet Metadata (File)");

        try
        {
            InputStream xssfWorkbookInputStream = new ByteArrayInputStream((byte[]) spreadsheetMap.get("data"));
            String      filename                = (String) spreadsheetMap.get("filename");
            String      location                = (String) spreadsheetMap.get("location");
            String      metadata                = generateXSSFSpeadsheetMetadata(baseRDFURI, xssfWorkbookInputStream, filename, location);
            xssfWorkbookInputStream.close();

            return metadata;
        }
        catch (Throwable throwable)
        {
            logger.log(Level.WARNING, "Problem Generating during XSSF Speadsheet Metadata Scan (File)", throwable);

            return null;
        }
    }

    private String generateXSSFSpeadsheetMetadata(URI baseRDFURI, InputStream xssfWorkbookInputStream, String filename, String location)
    {
        logger.log(Level.FINE, "Generate XSSF Speadsheet Metadata");

        try
        {
            XSSFWorkbook xssfWorkbook = new XSSFWorkbook(xssfWorkbookInputStream);

            StringBuffer rdfText = new StringBuffer();
            rdfText.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n\n");
            rdfText.append("<rdf:RDF xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\" xmlns:rdfs=\"http://www.w3.org/2000/01/rdf-schema#\" xmlns:x=\"http://rdfs.arjuna.com/xssf#\" xmlns:d=\"http://rdfs.arjuna.com/description#\">\n");

            boolean firstItem = true;

            List<String> sheetIds = new LinkedList<String>();
            for (int sheetIndex = 0; sheetIndex < xssfWorkbook.getNumberOfSheets(); sheetIndex++)
            {
                String sheetId = generateXSSFSheetMetadata(rdfText, firstItem, baseRDFURI, xssfWorkbook.getSheetAt(sheetIndex));
                if (sheetId != null)
                    sheetIds.add(sheetId);
                firstItem = firstItem && sheetIds.isEmpty();
            }

            String workbookId = UUID.randomUUID().toString();
            if (! firstItem)
                rdfText.append('\n');
            else
                rdfText.append('\n');
            rdfText.append("    <x:Workbook rdf:about=\"");
            rdfText.append(baseRDFURI.resolve('#' + workbookId));
            rdfText.append("\">\n");
            if (filename != null)
            {
                rdfText.append("        <d:hasTitle>");
                rdfText.append(filename);
                rdfText.append("</d:hasTitle>\n");
            }
            if (filename != null)
            {
                rdfText.append("        <d:hasSummary>");
                rdfText.append("This information was generated, automatically, from the spreadsheet ");
                rdfText.append(filename);
                rdfText.append("</d:hasSummary>\n");
            }
            if (location != null)
            {
                rdfText.append("        <d:hasLocation>");
                rdfText.append(location);
                rdfText.append("</d:hasLocation>\n");
            }
            for (String sheetId: sheetIds)
            {
                rdfText.append("        <x:hasSheet rdf:resource=\"");
                rdfText.append(baseRDFURI.resolve('#' + sheetId.toString()));
                rdfText.append("\"/>\n");
            }
            rdfText.append("    </x:Workbook>\n");

            rdfText.append("</rdf:RDF>\n");

            if (logger.isLoggable(Level.FINEST))
                logger.log(Level.FINEST, "XSSF Speadsheet RDF:\n[\n" + rdfText.toString() + "]");

            return rdfText.toString();
        }
        catch (Throwable throwable)
        {
            logger.log(Level.WARNING, "Problem Generating during XSSF Speadsheet Metadata Scan", throwable);

{'fraction_non_alphanumeric': 0.11442924923248675, 'fraction_numerical': 0.005209786956926226, 'mean_word_length': 2.3036263060848188, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 2, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4829759', 'n_tokens_mistral': 3249, 'n_tokens_neox': 3098, 'n_words': 563}