<commit_before>/*************************************************************************
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * Copyright 2000, 2010 Oracle and/or its affiliates.
 *
 * OpenOffice.org - a multi-platform office productivity suite
 *
 * This file is part of OpenOffice.org.
 *
 * OpenOffice.org is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License version 3
 * only, as published by the Free Software Foundation.
 *
 * OpenOffice.org is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License version 3 for more details
 * (a copy is included in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU Lesser General Public License
 * version 3 along with OpenOffice.org.  If not, see
 * <http://www.openoffice.org/license.html>
 * for a copy of the LGPLv3 License.
 *
 ************************************************************************/

// MARKER(update_precomp.py): autogen include statement, do not remove
#include "precompiled_linguistic.hxx"

#include <cppuhelper/factory.hxx>
#include <dicimp.hxx>
#include <hyphdsp.hxx>
#include <i18npool/lang.h>
#include <i18npool/mslangid.hxx>
#include <osl/mutex.hxx>
#include <tools/debug.hxx>
#include <tools/fsys.hxx>
#include <tools/stream.hxx>
#include <tools/string.hxx>
#include <tools/urlobj.hxx>
#include <unotools/processfactory.hxx>
#include <unotools/ucbstreamhelper.hxx>

#include <com/sun/star/ucb/XSimpleFileAccess.hpp>
#include <com/sun/star/linguistic2/DictionaryType.hpp>
#include <com/sun/star/linguistic2/DictionaryEventFlags.hpp>
#include <com/sun/star/registry/XRegistryKey.hpp>
#include <com/sun/star/io/XInputStream.hpp>
#include <com/sun/star/io/XOutputStream.hpp>

#include "defs.hxx"


using namespace utl;
using namespace osl;
using namespace rtl;
using namespace com::sun::star;
using namespace com::sun::star::lang;
using namespace com::sun::star::uno;
using namespace com::sun::star::linguistic2;
using namespace linguistic;

///////////////////////////////////////////////////////////////////////////

#define BUFSIZE             4096
#define VERS2_NOLANGUAGE    1024

#define MAX_HEADER_LENGTH 16

static const sal_Char*      pDicExt     = "dic";
static const sal_Char*      pVerStr2    = "WBSWG2";
static const sal_Char*      pVerStr5    = "WBSWG5";
static const sal_Char*      pVerStr6    = "WBSWG6";
static const sal_Char*      pVerOOo7    = "OOoUserDict1";

static const INT16 DIC_VERSION_DONTKNOW = -1;
static const INT16 DIC_VERSION_2 = 2;
static const INT16 DIC_VERSION_5 = 5;
static const INT16 DIC_VERSION_6 = 6;
static const INT16 DIC_VERSION_7 = 7;

static sal_Bool getTag(const ByteString &rLine,
        const sal_Char *pTagName, ByteString &rTagValue)
{
    xub_StrLen nPos = rLine.Search( pTagName );
    if (nPos == STRING_NOTFOUND)
        return FALSE;

    rTagValue = rLine.Copy( nPos + sal::static_int_cast< xub_StrLen >(strlen( pTagName )) ).EraseLeadingAndTrailingChars();
    return TRUE;
}


INT16 ReadDicVersion( SvStreamPtr &rpStream, USHORT &nLng, BOOL &bNeg )
{
    // Sniff the header
    INT16 nDicVersion = DIC_VERSION_DONTKNOW;
    sal_Char pMagicHeader[MAX_HEADER_LENGTH];

    nLng = LANGUAGE_NONE;
    bNeg = FALSE;

    if (!rpStream.get() || rpStream->GetError())
        return -1;

    sal_Size nSniffPos = rpStream->Tell();
    static sal_Size nVerOOo7Len = sal::static_int_cast< sal_Size >(strlen( pVerOOo7 ));
    pMagicHeader[ nVerOOo7Len ] = '\0';
    if ((rpStream->Read((void *) pMagicHeader, nVerOOo7Len) == nVerOOo7Len) &&
        !strcmp(pMagicHeader, pVerOOo7))
    {
        sal_Bool bSuccess;
        ByteString aLine;

        nDicVersion = DIC_VERSION_7;

        // 1st skip magic / header line
        rpStream->ReadLine(aLine);

        // 2nd line: language all | en-US | pt-BR ...
        while (sal_True == (bSuccess = rpStream->ReadLine(aLine)))
        {
            ByteString aTagValue;

            if (aLine.GetChar(0) == '#') // skip comments
                continue;

            // lang: field
            if (getTag(aLine, "lang: ", aTagValue))
            {
                if (aTagValue == "<none>")
                    nLng = LANGUAGE_NONE;
                else
                    nLng = MsLangId::convertIsoStringToLanguage(OUString(aTagValue.GetBuffer(),
                                aTagValue.Len(), RTL_TEXTENCODING_ASCII_US));
            }

            // type: negative / positive
            if (getTag(aLine, "type: ", aTagValue))
            {
                if (aTagValue == "negative")
                    bNeg = TRUE;
                else
                    bNeg = FALSE;
            }

            if (aLine.Search ("---") != STRING_NOTFOUND) // end of header
                break;
        }
        if (!bSuccess)
            return -2;
    }
    else
    {
        USHORT nLen;

        rpStream->Seek (nSniffPos );

        *rpStream >> nLen;
        if (nLen >= MAX_HEADER_LENGTH)
            return -1;

        rpStream->Read(pMagicHeader, nLen);
        pMagicHeader[nLen] = '\0';

        // Check version magic
        if (0 == strcmp( pMagicHeader, pVerStr6 ))
            nDicVersion = DIC_VERSION_6;
        else if (0 == strcmp( pMagicHeader, pVerStr5 ))
            nDicVersion = DIC_VERSION_5;
        else if (0 == strcmp( pMagicHeader, pVerStr2 ))
            nDicVersion = DIC_VERSION_2;
        else
            nDicVersion = DIC_VERSION_DONTKNOW;

        if (DIC_VERSION_2 == nDicVersion ||
            DIC_VERSION_5 == nDicVersion ||
            DIC_VERSION_6 == nDicVersion)
        {
            // The language of the dictionary
            *rpStream >> nLng;

            if (VERS2_NOLANGUAGE == nLng)
                nLng = LANGUAGE_NONE;

            // Negative Flag
            sal_Char nTmp;
            *rpStream >> nTmp;
            bNeg = (BOOL)nTmp;
        }
    }

    return nDicVersion;
}



const String GetDicExtension()
{
    return String::CreateFromAscii( pDicExt );
}

///////////////////////////////////////////////////////////////////////////

DictionaryNeo::DictionaryNeo() :
    aDicEvtListeners( GetLinguMutex() ),
    eDicType        (DictionaryType_POSITIVE),
    nLanguage       (LANGUAGE_NONE)
{
    nCount       = 0;
    nDicVersion  = DIC_VERSION_DONTKNOW;
    bNeedEntries = FALSE;
    bIsModified  = bIsActive = FALSE;
    bIsReadonly  = FALSE;
}

DictionaryNeo::DictionaryNeo(const OUString &rName,
                             INT16 nLang, DictionaryType eType,
                             const OUString &rMainURL,
                             BOOL bWriteable) :
    aDicEvtListeners( GetLinguMutex() ),
    aDicName        (rName),
    aMainURL        (rMainURL),
    eDicType        (eType),
    nLanguage       (nLang)
{
    nCount       = 0;
    nDicVersion  = DIC_VERSION_DONTKNOW;
    bNeedEntries = TRUE;
    bIsModified  = bIsActive = FALSE;
    bIsReadonly = !bWriteable;

    if( rMainURL.getLength() > 0 )
    {
        BOOL bExists = FileExists( rMainURL );
        if( !bExists )
        {
            // save new dictionaries with in Format 7 (UTF8 plain text)
            nDicVersion  = 7;<commit_msg>cws tl80: #i106032# changed default dictionary type for new user dictionaries to plain UTF-8 text file<commit_after>/*************************************************************************
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * Copyright 2000, 2010 Oracle and/or its affiliates.
 *
 * OpenOffice.org - a multi-platform office productivity suite
 *
 * This file is part of OpenOffice.org.
 *
 * OpenOffice.org is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License version 3
 * only, as published by the Free Software Foundation.
 *
 * OpenOffice.org is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License version 3 for more details
 * (a copy is included in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU Lesser General Public License
 * version 3 along with OpenOffice.org.  If not, see
 * <http://www.openoffice.org/license.html>
 * for a copy of the LGPLv3 License.
 *
 ************************************************************************/

// MARKER(update_precomp.py): autogen include statement, do not remove
#include "precompiled_linguistic.hxx"

#include <cppuhelper/factory.hxx>
#include <dicimp.hxx>
#include <hyphdsp.hxx>
#include <i18npool/lang.h>
#include <i18npool/mslangid.hxx>
#include <osl/mutex.hxx>
#include <tools/debug.hxx>
#include <tools/fsys.hxx>
#include <tools/stream.hxx>
#include <tools/string.hxx>
#include <tools/urlobj.hxx>
#include <unotools/processfactory.hxx>
#include <unotools/ucbstreamhelper.hxx>

#include <com/sun/star/ucb/XSimpleFileAccess.hpp>
#include <com/sun/star/linguistic2/DictionaryType.hpp>
#include <com/sun/star/linguistic2/DictionaryEventFlags.hpp>
#include <com/sun/star/registry/XRegistryKey.hpp>
#include <com/sun/star/io/XInputStream.hpp>
#include <com/sun/star/io/XOutputStream.hpp>

#include "defs.hxx"


using namespace utl;
using namespace osl;
using namespace rtl;
using namespace com::sun::star;
using namespace com::sun::star::lang;
using namespace com::sun::star::uno;
using namespace com::sun::star::linguistic2;
using namespace linguistic;

///////////////////////////////////////////////////////////////////////////

#define BUFSIZE             4096
#define VERS2_NOLANGUAGE    1024

#define MAX_HEADER_LENGTH 16

static const sal_Char*      pDicExt     = "dic";
static const sal_Char*      pVerStr2    = "WBSWG2";
static const sal_Char*      pVerStr5    = "WBSWG5";
static const sal_Char*      pVerStr6    = "WBSWG6";
static const sal_Char*      pVerOOo7    = "OOoUserDict1";

static const INT16 DIC_VERSION_DONTKNOW = -1;
static const INT16 DIC_VERSION_2 = 2;
static const INT16 DIC_VERSION_5 = 5;
static const INT16 DIC_VERSION_6 = 6;
static const INT16 DIC_VERSION_7 = 7;

static sal_Bool getTag(const ByteString &rLine,
        const sal_Char *pTagName, ByteString &rTagValue)
{
    xub_StrLen nPos = rLine.Search( pTagName );
    if (nPos == STRING_NOTFOUND)
        return FALSE;

    rTagValue = rLine.Copy( nPos + sal::static_int_cast< xub_StrLen >(strlen( pTagName )) ).EraseLeadingAndTrailingChars();
    return TRUE;
}


INT16 ReadDicVersion( SvStreamPtr &rpStream, USHORT &nLng, BOOL &bNeg )
{
    // Sniff the header
    INT16 nDicVersion = DIC_VERSION_DONTKNOW;
    sal_Char pMagicHeader[MAX_HEADER_LENGTH];

    nLng = LANGUAGE_NONE;
    bNeg = FALSE;

    if (!rpStream.get() || rpStream->GetError())
        return -1;

    sal_Size nSniffPos = rpStream->Tell();
    static sal_Size nVerOOo7Len = sal::static_int_cast< sal_Size >(strlen( pVerOOo7 ));
    pMagicHeader[ nVerOOo7Len ] = '\0';
    if ((rpStream->Read((void *) pMagicHeader, nVerOOo7Len) == nVerOOo7Len) &&
        !strcmp(pMagicHeader, pVerOOo7))
    {
        sal_Bool bSuccess;
        ByteString aLine;

        nDicVersion = DIC_VERSION_7;

        // 1st skip magic / header line
        rpStream->ReadLine(aLine);

        // 2nd line: language all | en-US | pt-BR ...
        while (sal_True == (bSuccess = rpStream->ReadLine(aLine)))
        {
            ByteString aTagValue;

            if (aLine.GetChar(0) == '#') // skip comments
                continue;

            // lang: field
            if (getTag(aLine, "lang: ", aTagValue))
            {
                if (aTagValue == "<none>")
                    nLng = LANGUAGE_NONE;
                else
                    nLng = MsLangId::convertIsoStringToLanguage(OUString(aTagValue.GetBuffer(),
                                aTagValue.Len(), RTL_TEXTENCODING_ASCII_US));
            }

            // type: negative / positive
            if (getTag(aLine, "type: ", aTagValue))
            {
                if (aTagValue == "negative")
                    bNeg = TRUE;
                else
                    bNeg = FALSE;
            }

            if (aLine.Search ("---") != STRING_NOTFOUND) // end of header
                break;
        }
        if (!bSuccess)
            return -2;
    }
    else
    {
        USHORT nLen;

        rpStream->Seek (nSniffPos );

        *rpStream >> nLen;
        if (nLen >= MAX_HEADER_LENGTH)
            return -1;

        rpStream->Read(pMagicHeader, nLen);
        pMagicHeader[nLen] = '\0';

        // Check version magic
        if (0 == strcmp( pMagicHeader, pVerStr6 ))
            nDicVersion = DIC_VERSION_6;
        else if (0 == strcmp( pMagicHeader, pVerStr5 ))
            nDicVersion = DIC_VERSION_5;
        else if (0 == strcmp( pMagicHeader, pVerStr2 ))
            nDicVersion = DIC_VERSION_2;
        else
            nDicVersion = DIC_VERSION_DONTKNOW;

        if (DIC_VERSION_2 == nDicVersion ||
            DIC_VERSION_5 == nDicVersion ||
            DIC_VERSION_6 == nDicVersion)
        {
            // The language of the dictionary
            *rpStream >> nLng;

            if (VERS2_NOLANGUAGE == nLng)
                nLng = LANGUAGE_NONE;

            // Negative Flag
            sal_Char nTmp;
            *rpStream >> nTmp;
            bNeg = (BOOL)nTmp;
        }
    }

    return nDicVersion;
}



const String GetDicExtension()
{
    return String::CreateFromAscii( pDicExt );
}

///////////////////////////////////////////////////////////////////////////

DictionaryNeo::DictionaryNeo() :
    aDicEvtListeners( GetLinguMutex() ),
    eDicType        (DictionaryType_POSITIVE),
    nLanguage       (LANGUAGE_NONE)
{
    nCount       = 0;
    nDicVersion  = DIC_VERSION_DONTKNOW;
    bNeedEntries = FALSE;
    bIsModified  = bIsActive = FALSE;
    bIsReadonly  = FALSE;
}

DictionaryNeo::DictionaryNeo(const OUString &rName,
                             INT16 nLang, DictionaryType eType,
                             const OUString &rMainURL,
                             BOOL bWriteable) :
    aDicEvtListeners( GetLinguMutex() ),
    aDicName        (rName),
    aMainURL        (rMainURL),
    eDicType        (eType),
    nLanguage       (nLang)
{
    nCount       = 0;
    nDicVersion  = DIC_VERSION_DONTKNOW;
    bNeedEntries = TRUE;
    bIsModified  = bIsActive = FALSE;
    bIsReadonly = !bWriteable;

    if( rMainURL.getLength() > 0 )
    {
        BOOL bExists = FileExists( rMainURL );
        if( !bExists )
        {
            // save new dictionaries with in Format 7 (UTF8 plain text)
            nDicVersion  = DIC_VERSION_7;
{'fraction_non_alphanumeric': 0.13511147033070653, 'fraction_numerical': 0.013672795851013672, 'mean_word_length': 2.275534965806309, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 79, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2947166', 'n_tokens_mistral': 4880, 'n_tokens_neox': 4649, 'n_words': 1158}