<commit_before>/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*
 * This file is part of the LibreOffice project.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

#include <writerperfect/WPXSvInputStream.hxx>

#include <com/sun/star/packages/zip/XZipFileAccess2.hpp>
#include <com/sun/star/uno/Any.hxx>

#include <comphelper/processfactory.hxx>
#include <comphelper/seekableinput.hxx>

#include <rtl/string.hxx>

#include <sot/storage.hxx>

#include <tools/stream.hxx>
#include <unotools/streamwrap.hxx>
#include <unotools/ucbstreamhelper.hxx>

#include <limits>
#include <vector>

#include <boost/noncopyable.hpp>
#include <boost/scoped_ptr.hpp>
#include <boost/unordered_map.hpp>

namespace writerperfect
{

using namespace ::com::sun::star::uno;
using namespace ::com::sun::star::io;

namespace container = com::sun::star::container;
namespace lang = com::sun::star::lang;
namespace packages = com::sun::star::packages;

namespace
{

class PositionHolder : boost::noncopyable
{
public:
    explicit PositionHolder(const Reference<XSeekable> &rxSeekable);
    ~PositionHolder();

private:
    const Reference<XSeekable> mxSeekable;
    const sal_uInt64 mnPosition;
};

PositionHolder::PositionHolder(const Reference<XSeekable> &rxSeekable)
    : mxSeekable(rxSeekable)
    , mnPosition(rxSeekable->getPosition())
{
}

PositionHolder::~PositionHolder() try
{
    mxSeekable->seek(mnPosition);
}
catch (...)
{
}

} // anonymous namespace

typedef struct
{
    SotStorageRef ref;
} SotStorageRefWrapper;

typedef struct
{
    SotStorageStreamRef ref;
} SotStorageStreamRefWrapper;

namespace
{

const rtl::OUString concatPath(const rtl::OUString &lhs, const rtl::OUString &rhs)
{
    if (lhs.isEmpty())
        return rhs;
    return lhs + "/" + rhs;
}

struct OLEStreamData
{
    explicit OLEStreamData(const rtl::OString &rName);

    SotStorageStreamRefWrapper stream;

    /** Name of the stream.
      *
      * This is not @c rtl::OUString, because we need to be able to
      * produce const char* from it.
      */
    rtl::OString name;
};

typedef boost::unordered_map<rtl::OUString, std::size_t, rtl::OUStringHash> NameMap_t;
typedef boost::unordered_map<rtl::OUString, SotStorageRefWrapper, rtl::OUStringHash> OLEStorageMap_t;

/** Representation of an OLE2 storage.
  *
  * This class tries to bring a bit of sanity to use of SotStorage with
  * respect to the needs of @c librevenge::RVNGInputStream API. It
  * holds all nested storages for the whole lifetime (more precisely,
  * since initialization, which is performed by calling @c
  * initialize()), thus ensuring that no created stream is destroyed
  * just because its parent storage went out of scope. It also holds a
  * bidirectional map of stream names to their indexes (index of a
  * stream is determined by deep-first traversal), which is also
  * populated during initialization (member variables @c maStreams and
  * @c maNameMap).
  *
  * Streams are created on demand (and saved, for the same reason as
  * storages).
  */
struct OLEStorageImpl
{
    OLEStorageImpl();

    void initialize(SvStream *pStream);

    SotStorageStreamRef getStream(const rtl::OUString &rPath);
    SotStorageStreamRef getStream(std::size_t nId);

private:
    void traverse(const SotStorageRef &rStorage, const rtl::OUString &rPath);

    SotStorageStreamRef createStream(const rtl::OUString &rPath);

public:
    SotStorageRefWrapper mxRootStorage; //< root storage of the OLE2
    OLEStorageMap_t maStorageMap; //< map of all sub storages by name
    ::std::vector< OLEStreamData > maStreams; //< list of streams and their names
    NameMap_t maNameMap; //< map of stream names to indexes (into @c maStreams)
    bool mbInitialized;
};

OLEStreamData::OLEStreamData(const rtl::OString &rName)
    : stream()
    , name(rName)
{
}

OLEStorageImpl::OLEStorageImpl()
    : mxRootStorage()
    , maStorageMap()
    , maStreams()
    , maNameMap()
    , mbInitialized(false)
{
}

void OLEStorageImpl::initialize(SvStream *const pStream)
{
    if (!pStream)
        return;

    mxRootStorage.ref = new SotStorage( pStream, true );

    traverse(mxRootStorage.ref, "");

    mbInitialized = true;
}

SotStorageStreamRef OLEStorageImpl::getStream(const rtl::OUString &rPath)
{
    rtl::OUString aPath(rPath);
    // accept paths which begin by '/'
    if (aPath.startsWith("/") && aPath.getLength() >= 2)
        aPath=rPath.copy(1);
    NameMap_t::iterator aIt = maNameMap.find(aPath);

    // For the while don't return stream in this situation.
    // Later, given how libcdr's zip stream implementation behaves,
    // return the first stream in the storage if there is one.
    if (maNameMap.end() == aIt)
        return SotStorageStreamRef();

    if (!maStreams[aIt->second].stream.ref.Is())
        maStreams[aIt->second].stream.ref = createStream(aPath);

    return maStreams[aIt->second].stream.ref;
}

SotStorageStreamRef OLEStorageImpl::getStream(const std::size_t nId)
{
    if (!maStreams[nId].stream.ref.Is())
        maStreams[nId].stream.ref = createStream(rtl::OStringToOUString(maStreams[nId].name, RTL_TEXTENCODING_UTF8));

    return maStreams[nId].stream.ref;
}

void OLEStorageImpl::traverse(const SotStorageRef &rStorage, const rtl::OUString &rPath)
{
    SvStorageInfoList infos;

    rStorage->FillInfoList(&infos);

    for (SvStorageInfoList::const_iterator aIt = infos.begin(); infos.end() != aIt; ++aIt)
    {
        if (aIt->IsStream())
        {
            maStreams.push_back(OLEStreamData(rtl::OUStringToOString(concatPath(rPath, aIt->GetName()), RTL_TEXTENCODING_UTF8)));
            maNameMap[concatPath(rPath, aIt->GetName())] = maStreams.size() - 1;
        }
        else if (aIt->IsStorage())
        {
            const rtl::OUString aPath = concatPath(rPath, aIt->GetName());
            SotStorageRefWrapper xStorage;
            xStorage.ref = rStorage->OpenSotStorage(aIt->GetName(), STREAM_STD_READ);
            maStorageMap[aPath] = xStorage;

            // deep-first traversal
            traverse(xStorage.ref, aPath);
        }
        else
        {
            assert(false);
        }
    }
}

SotStorageStreamRef OLEStorageImpl::createStream(const rtl::OUString &rPath)
{
    const sal_Int32 nDelim = rPath.lastIndexOf(sal_Unicode('/'));

    if (-1 == nDelim)
        return mxRootStorage.ref->OpenSotStream(rPath, STREAM_STD_READ);

    const rtl::OUString aDir = rPath.copy(0, nDelim);
    const rtl::OUString aName = rPath.copy(nDelim + 1);

    const OLEStorageMap_t::const_iterator aIt = maStorageMap.find(aDir);

    // We can only get there for paths that are present in the OLE.
    // Which means the storage must exist.
    assert(maStorageMap.end() != aIt);

    return aIt->second.ref->OpenSotStream(aName, STREAM_STD_READ);
}

}

namespace
{

struct ZipStreamData
{
    explicit ZipStreamData(const rtl::OString &rName);

    Reference<XInputStream> xStream;

    /** Name of the stream.
      *
      * This is not @c rtl::OUString, because we need to be able to
      * produce const char* from it.
      */
    rtl::OString aName;
};

/** Representation of a Zip storage.
  *
  * This is quite similar to OLEStorageImpl, except that we do not need
  * to keep all storages (folders) open.
  */
struct ZipStorageImpl
{
    ZipStorageImpl(const Reference<container::XNameAccess> &rxContainer);

    /** Initialize for access.
      *
      * This creates a bidirectional map of stream names to their
      * indexes (index of a stream is determined by deep-first
      * traversal).
      */
    void initialize();

    Reference<XInputStream> getStream(const rtl::OUString &rPath);
    Reference<XInputStream> getStream(std::size_t nId);

private:
    void traverse(const Reference<container::XNameAccess> &rxEnum);

    Reference<XInputStream> createStream(const rtl::OUString &rPath);

public:
    Reference<container::XNameAccess> mxContainer; //< root of the Zip
    ::std::vector< ZipStreamData > maStreams; //< list of streams and their names
    NameMap_t maNameMap; //< map of stream names to indexes (into @c maStreams)
    bool mbInitialized;
};

ZipStreamData::ZipStreamData(const rtl::OString &rName)
    : xStream()
    , aName(rName)
{
}

ZipStorageImpl::ZipStorageImpl(const Reference<container::XNameAccess> &rxContainer)
    : mxContainer(rxContainer)
    , maStreams()
    , maNameMap()
    , mbInitialized(false)
{
    assert(mxContainer.is());
}

void ZipStorageImpl::initialize()
{
    traverse(mxContainer);

    mbInitialized = true;
}

Reference<XInputStream> ZipStorageImpl::getStream(const rtl::OUString &rPath)
{
    NameMap_t::iterator aIt = maNameMap.find(rPath);

    // For the while don't return stream in this situation.
    // Later, given how libcdr's zip stream implementation behaves,
    // return the first stream in the storage if there is one.
    if (maNameMap.end() == aIt)
        return Reference<XInputStream>();

    if (!maStreams[aIt->second].xStream.is())
        maStreams[aIt->second].xStream = createStream(rPath);

    return maStreams[aIt->second].xStream;
}

Reference<XInputStream> ZipStorageImpl::getStream(const std::size_t nId)
{
    if (!maStreams[nId].xStream.is())
        maStreams[nId].xStream = createStream(rtl::OStringToOUString(maStreams[nId].aName, RTL_TEXTENCODING_UTF8));

    return maStreams[nId].xStream;
}

void ZipStorageImpl::traverse(const Reference<container::XNameAccess> &rxContainer)
{
    const Sequence<rtl::OUString> lNames = rxContainer->getElementNames();
<commit_msg>generalize this for Zip too<commit_after>/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*
 * This file is part of the LibreOffice project.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

#include <writerperfect/WPXSvInputStream.hxx>

#include <com/sun/star/packages/zip/XZipFileAccess2.hpp>
#include <com/sun/star/uno/Any.hxx>

#include <comphelper/processfactory.hxx>
#include <comphelper/seekableinput.hxx>

#include <rtl/string.hxx>

#include <sot/storage.hxx>

#include <tools/stream.hxx>
#include <unotools/streamwrap.hxx>
#include <unotools/ucbstreamhelper.hxx>

#include <limits>
#include <vector>

#include <boost/noncopyable.hpp>
#include <boost/scoped_ptr.hpp>
#include <boost/unordered_map.hpp>

namespace writerperfect
{

using namespace ::com::sun::star::uno;
using namespace ::com::sun::star::io;

namespace container = com::sun::star::container;
namespace lang = com::sun::star::lang;
namespace packages = com::sun::star::packages;

namespace
{

class PositionHolder : boost::noncopyable
{
public:
    explicit PositionHolder(const Reference<XSeekable> &rxSeekable);
    ~PositionHolder();

private:
    const Reference<XSeekable> mxSeekable;
    const sal_uInt64 mnPosition;
};

PositionHolder::PositionHolder(const Reference<XSeekable> &rxSeekable)
    : mxSeekable(rxSeekable)
    , mnPosition(rxSeekable->getPosition())
{
}

PositionHolder::~PositionHolder() try
{
    mxSeekable->seek(mnPosition);
}
catch (...)
{
}

} // anonymous namespace

typedef struct
{
    SotStorageRef ref;
} SotStorageRefWrapper;

typedef struct
{
    SotStorageStreamRef ref;
} SotStorageStreamRefWrapper;

namespace
{

rtl::OUString lcl_normalizeSubStreamPath(const rtl::OUString &rPath)
{
    // accept paths which begin by '/'
    // TODO: maybe this should to a full normalization
    if (rPath.startsWith("/") && rPath.getLength() >= 2)
        return rPath.copy(1);
    return rPath;
}

}

namespace
{

const rtl::OUString concatPath(const rtl::OUString &lhs, const rtl::OUString &rhs)
{
    if (lhs.isEmpty())
        return rhs;
    return lhs + "/" + rhs;
}

struct OLEStreamData
{
    explicit OLEStreamData(const rtl::OString &rName);

    SotStorageStreamRefWrapper stream;

    /** Name of the stream.
      *
      * This is not @c rtl::OUString, because we need to be able to
      * produce const char* from it.
      */
    rtl::OString name;
};

typedef boost::unordered_map<rtl::OUString, std::size_t, rtl::OUStringHash> NameMap_t;
typedef boost::unordered_map<rtl::OUString, SotStorageRefWrapper, rtl::OUStringHash> OLEStorageMap_t;

/** Representation of an OLE2 storage.
  *
  * This class tries to bring a bit of sanity to use of SotStorage with
  * respect to the needs of @c librevenge::RVNGInputStream API. It
  * holds all nested storages for the whole lifetime (more precisely,
  * since initialization, which is performed by calling @c
  * initialize()), thus ensuring that no created stream is destroyed
  * just because its parent storage went out of scope. It also holds a
  * bidirectional map of stream names to their indexes (index of a
  * stream is determined by deep-first traversal), which is also
  * populated during initialization (member variables @c maStreams and
  * @c maNameMap).
  *
  * Streams are created on demand (and saved, for the same reason as
  * storages).
  */
struct OLEStorageImpl
{
    OLEStorageImpl();

    void initialize(SvStream *pStream);

    SotStorageStreamRef getStream(const rtl::OUString &rPath);
    SotStorageStreamRef getStream(std::size_t nId);

private:
    void traverse(const SotStorageRef &rStorage, const rtl::OUString &rPath);

    SotStorageStreamRef createStream(const rtl::OUString &rPath);

public:
    SotStorageRefWrapper mxRootStorage; //< root storage of the OLE2
    OLEStorageMap_t maStorageMap; //< map of all sub storages by name
    ::std::vector< OLEStreamData > maStreams; //< list of streams and their names
    NameMap_t maNameMap; //< map of stream names to indexes (into @c maStreams)
    bool mbInitialized;
};

OLEStreamData::OLEStreamData(const rtl::OString &rName)
    : stream()
    , name(rName)
{
}

OLEStorageImpl::OLEStorageImpl()
    : mxRootStorage()
    , maStorageMap()
    , maStreams()
    , maNameMap()
    , mbInitialized(false)
{
}

void OLEStorageImpl::initialize(SvStream *const pStream)
{
    if (!pStream)
        return;

    mxRootStorage.ref = new SotStorage( pStream, true );

    traverse(mxRootStorage.ref, "");

    mbInitialized = true;
}

SotStorageStreamRef OLEStorageImpl::getStream(const rtl::OUString &rPath)
{
    const rtl::OUString aPath(lcl_normalizeSubStreamPath(rPath));
    NameMap_t::iterator aIt = maNameMap.find(aPath);

    // For the while don't return stream in this situation.
    // Later, given how libcdr's zip stream implementation behaves,
    // return the first stream in the storage if there is one.
    if (maNameMap.end() == aIt)
        return SotStorageStreamRef();

    if (!maStreams[aIt->second].stream.ref.Is())
        maStreams[aIt->second].stream.ref = createStream(aPath);

    return maStreams[aIt->second].stream.ref;
}

SotStorageStreamRef OLEStorageImpl::getStream(const std::size_t nId)
{
    if (!maStreams[nId].stream.ref.Is())
        maStreams[nId].stream.ref = createStream(rtl::OStringToOUString(maStreams[nId].name, RTL_TEXTENCODING_UTF8));

    return maStreams[nId].stream.ref;
}

void OLEStorageImpl::traverse(const SotStorageRef &rStorage, const rtl::OUString &rPath)
{
    SvStorageInfoList infos;

    rStorage->FillInfoList(&infos);

    for (SvStorageInfoList::const_iterator aIt = infos.begin(); infos.end() != aIt; ++aIt)
    {
        if (aIt->IsStream())
        {
            maStreams.push_back(OLEStreamData(rtl::OUStringToOString(concatPath(rPath, aIt->GetName()), RTL_TEXTENCODING_UTF8)));
            maNameMap[concatPath(rPath, aIt->GetName())] = maStreams.size() - 1;
        }
        else if (aIt->IsStorage())
        {
            const rtl::OUString aPath = concatPath(rPath, aIt->GetName());
            SotStorageRefWrapper xStorage;
            xStorage.ref = rStorage->OpenSotStorage(aIt->GetName(), STREAM_STD_READ);
            maStorageMap[aPath] = xStorage;

            // deep-first traversal
            traverse(xStorage.ref, aPath);
        }
        else
        {
            assert(false);
        }
    }
}

SotStorageStreamRef OLEStorageImpl::createStream(const rtl::OUString &rPath)
{
    const sal_Int32 nDelim = rPath.lastIndexOf(sal_Unicode('/'));

    if (-1 == nDelim)
        return mxRootStorage.ref->OpenSotStream(rPath, STREAM_STD_READ);

    const rtl::OUString aDir = rPath.copy(0, nDelim);
    const rtl::OUString aName = rPath.copy(nDelim + 1);

    const OLEStorageMap_t::const_iterator aIt = maStorageMap.find(aDir);

    // We can only get there for paths that are present in the OLE.
    // Which means the storage must exist.
    assert(maStorageMap.end() != aIt);

    return aIt->second.ref->OpenSotStream(aName, STREAM_STD_READ);
}

}

namespace
{

struct ZipStreamData
{
    explicit ZipStreamData(const rtl::OString &rName);

    Reference<XInputStream> xStream;

    /** Name of the stream.
      *
      * This is not @c rtl::OUString, because we need to be able to
      * produce const char* from it.
      */
    rtl::OString aName;
};

/** Representation of a Zip storage.
  *
  * This is quite similar to OLEStorageImpl, except that we do not need
  * to keep all storages (folders) open.
  */
struct ZipStorageImpl
{
    ZipStorageImpl(const Reference<container::XNameAccess> &rxContainer);

    /** Initialize for access.
      *
      * This creates a bidirectional map of stream names to their
      * indexes (index of a stream is determined by deep-first
      * traversal).
      */
    void initialize();

    Reference<XInputStream> getStream(const rtl::OUString &rPath);
    Reference<XInputStream> getStream(std::size_t nId);

private:
    void traverse(const Reference<container::XNameAccess> &rxEnum);

    Reference<XInputStream> createStream(const rtl::OUString &rPath);

public:
    Reference<container::XNameAccess> mxContainer; //< root of the Zip
    ::std::vector< ZipStreamData > maStreams; //< list of streams and their names
    NameMap_t maNameMap; //< map of stream names to indexes (into @c maStreams)
    bool mbInitialized;
};

ZipStreamData::ZipStreamData(const rtl::OString &rName)
    : xStream()
    , aName(rName)
{
}

ZipStorageImpl::ZipStorageImpl(const Reference<container::XNameAccess> &rxContainer)
    : mxContainer(rxContainer)
    , maStreams()
    , maNameMap()
    , mbInitialized(false)
{
    assert(mxContainer.is());
}

void ZipStorageImpl::initialize()
{
    traverse(mxContainer);

    mbInitialized = true;
}

Reference<XInputStream> ZipStorageImpl::getStream(const rtl::OUString &rPath)
{
    const rtl::OUString aPath(lcl_normalizeSubStreamPath(rPath));
    NameMap_t::iterator aIt = maNameMap.find(aPath);

    // For the while don't return stream in this situation.
    // Later, given how libcdr's zip stream implementation behaves,
    // return the first stream in the storage if there is one.
    if (maNameMap.end() == aIt)
        return Reference<XInputStream>();

    if (!maStreams[aIt->second].xStream.is())
        maStreams[aIt->second].xStream = createStream(aPath);

    return maStreams[aIt->second].xStream;
}

Reference<XInputStream> ZipStorageImpl::getStream(const std::size_t nId)
{
    if (!maStreams[nId].xStream.is())
        maStreams[nId].xStream = createStream(rtl::OStringToOUString(maStreams[nId].aName, RTL_TEXTENCODING_UTF8));

    return maStreams[nId].xStream;
}

void ZipStorageImpl::traverse(const Reference<container::XNameAccess> &rxContainer)
{
    const Sequence<rtl::OUString> lNames = rxContainer->getElementNames();

{'fraction_non_alphanumeric': 0.11784768886173895, 'fraction_numerical': 0.002239869680309509, 'mean_word_length': 3.842247966477693, 'pattern_counts': {'":': 0, '<': 87, '<?xml version=': 0, '>': 115, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6639994', 'n_tokens_mistral': 6560, 'n_tokens_neox': 6346, 'n_words': 1574}