<commit_before>## $Id$

## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

__revision__ = "$Id$"

## import interesting modules:
import string
import os
import sys
import time
import types
import re
import mimetypes
import shutil
import md5
import urllib
from xml.sax.saxutils import quoteattr

from invenio.config import \
     cdslang, \
     filedir, \
     filedirsize, \
     images, \
     storage, \
     version, \
     weburl
from invenio.access_control_engine import acc_authorize_action
from invenio.access_control_admin import acc_isRole
from invenio.webpage import page, create_error_box
from invenio.webuser import getUid, get_email
from invenio.dbquery import run_sql
from invenio.websubmit_config import *

from invenio.messages import gettext_set_language
import invenio.template
websubmit_templates = invenio.template.load('websubmit')

archivepath = filedir
archivesize = filedirsize

# sort compressed file extensions list to get lengthy ones first:
CFG_COMPRESSED_FILE_EXTENSIONS_SORTED = CFG_COMPRESSED_FILE_EXTENSIONS
CFG_COMPRESSED_FILE_EXTENSIONS_SORTED.sort()

def file_strip_ext(file):
    for c_ext in CFG_KNOWN_FILE_EXTENSIONS:
        if file[-len(c_ext):len(file)]==c_ext and file[-len(c_ext)-1]==".":
            file = file[0:-len(c_ext)-1]
    return file

class BibRecDocs:
    """this class represents all the files attached to one record"""
    def __init__(self,recid):
        self.id = recid
        self.bibdocs = []
        self.buildBibDocList()

    def buildBibDocList(self):
        self.bibdocs = []
        res = run_sql("select id_bibdoc,type,status from bibrec_bibdoc,bibdoc where id=id_bibdoc and id_bibrec=%s", (self.id,))
        for row in res:
            if row[2] == "":
                status = 0
            else:
                status = int(row[2])
            if status & 1 == 0:
                self.bibdocs.append(BibDoc(bibdocid=row[0],recid=self.id))

    def listBibDocs(self,type=""):
        tmp=[]
        for bibdoc in self.bibdocs:
            if type=="" or type == bibdoc.getType():
                tmp.append(bibdoc)
        return tmp

    def getBibDocNames(self,type="Main"):
        names = []
        for bibdoc in self.listBibDocs(type):
            names.append(bibdoc.getDocName())
        return names

    def getBibDoc(self,bibdocid):
        for bibdoc in self.bibdocs:
            if bibdoc.getId() == bibdocid:
                return bibdoc
        return None

    def deleteBibDoc(self,bibdocid):
        for bibdoc in self.bibdocs:
            if bibdoc.getId() == bibdocid:
                bibdoc.delete()
        self.buildBibDocList()

    def addBibDoc(self,type="Main",docname="file"):
        while docname in self.getBibDocNames(type):
            match = re.match("(.*_)([^_]*)",docname)
            if match:
                try:
                    docname = match.group(1)+str(int(match.group(2)) + 1)
                except:
                    docname = docname + "_2"
            else:
                docname = docname + "_2"
        bibdoc = BibDoc(recid=self.id,type=type,docname=docname)
        if bibdoc is not None:
            self.bibdocs.append(bibdoc)
        return bibdoc

    def addNewFile(self,fullpath,type="Main"):
        filename = re.sub("\..*","",re.sub(r".*[\\/:]", "", fullpath))
        bibdoc = self.addBibDoc(type,filename)
        if bibdoc is not None:
            bibdoc.addFilesNewVersion(files=[fullpath])
            return bibdoc
        return None

    def addNewVersion(self,fullpath,bibdocid):
        bibdoc = self.getBibDoc(bibdocid)
        if bibdoc is not None:
            bibdoc.addFilesNewVersion(files=[fullpath])
            docname = re.sub("\..*","",re.sub(r".*[\\/:]", "", fullpath))
            if docname != bibdoc.getDocName():
                while docname in self.getBibDocNames(bibdoc.getType()):
                    match = re.match("(.*_)([^_]*)",docname)
                    if match:
                        try:
                            docname = match.group(1)+str(int(match.group(2)) + 1)
                        except:
                            docname = docname + "_2"
                    else:
                        docname = docname + "_2"
                bibdoc.changeName(docname)
            return bibdoc
        return None

    def addNewFormat(self,fullpath,bibdocid):
        bibdoc = self.getBibDoc(bibdocid)
        if bibdoc is not None:
            bibdoc.addFilesNewFormat(files=[fullpath])
            return bibdoc
        return None

    def listLatestFiles(self,type=""):
        docfiles = []
        for bibdoc in self.listBibDocs(type):
            for docfile in bibdoc.listLatestFiles():
                docfiles.append(docfile)
        return docfiles

    def checkFileExists(self,fullpath,type=""):
        if os.path.exists(fullpath):
            docfiles = self.listLatestFiles(type)
            for docfile in docfiles:
                if md5.new(readfile(fullpath)).digest() == md5.new(readfile(docfile.getPath())).digest():
                    return docfile.getBibDocId()
        else:
            return 0

    def proposeUniqueName(self, filename, type=""):
        """Produces a filename starting from filename that is known to not
        overwrite files already existants. Returns the same filename if it's
        already unique, otherwise it returns name-#.ext where name, ext are
        part of filename and -# is a progressive integer number.
        #FIXME Still to be checked.
        """

        # Splitting name from ext in order to insert the incremental number
        if '.' in filename:
            [name, ext] = filename.split('.', 1)
            ext = '.' + ext
        else:
            name = filename
            ext = ''

        # Let's check if the filename is not already ok
        docfiles = self.listLatestFiles(type)
        ok_p = True
        for docfile in docfiles:
            if filename in docfile.name():
                ok_p = False
                break

        if not ok_p:
            # Let's try with incremental numbers
            n = 1
            while not ok_p:
                ok_p = True
                for docfile in docfiles:
                    filename = "%s-%i%s" % (name, n, ext)
                    if filename in docfile.name():
                        ok_p = False
                        break
                n += 1
        return filename

    def display(self,bibdocid="",version="",type="", ln = cdslang):
        t=""
        bibdocs = []
        if bibdocid!="":
            for bibdoc in self.bibdocs:
                if bibdoc.getId() == bibdocid:
                    bibdocs.append(bibdoc)
        else:
            bibdocs = self.listBibDocs(type)
        if len(bibdocs) > 0:
            types = listTypesFromArray(bibdocs)
            fulltypes = []
            for mytype in types:<commit_msg>Adding a bit of documentation...<commit_after>## $Id$

## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

__revision__ = "$Id$"

## import interesting modules:
import string
import os
import sys
import time
import types
import re
import mimetypes
import shutil
import md5
import urllib
from xml.sax.saxutils import quoteattr

from invenio.config import \
     cdslang, \
     filedir, \
     filedirsize, \
     images, \
     storage, \
     version, \
     weburl
from invenio.access_control_engine import acc_authorize_action
from invenio.access_control_admin import acc_isRole
from invenio.webpage import page, create_error_box
from invenio.webuser import getUid, get_email
from invenio.dbquery import run_sql
from invenio.websubmit_config import *

from invenio.messages import gettext_set_language
import invenio.template
websubmit_templates = invenio.template.load('websubmit')

archivepath = filedir
archivesize = filedirsize

# sort compressed file extensions list to get lengthy ones first:
CFG_COMPRESSED_FILE_EXTENSIONS_SORTED = CFG_COMPRESSED_FILE_EXTENSIONS
CFG_COMPRESSED_FILE_EXTENSIONS_SORTED.sort()

def file_strip_ext(file):
    for c_ext in CFG_KNOWN_FILE_EXTENSIONS:
        if file[-len(c_ext):len(file)]==c_ext and file[-len(c_ext)-1]==".":
            file = file[0:-len(c_ext)-1]
    return file

class BibRecDocs:
    """this class represents all the files attached to one record"""
    def __init__(self,recid):
        self.id = recid
        self.bibdocs = []
        self.buildBibDocList()

    def buildBibDocList(self):
        self.bibdocs = []
        res = run_sql("select id_bibdoc,type,status from bibrec_bibdoc,bibdoc where id=id_bibdoc and id_bibrec=%s", (self.id,))
        for row in res:
            if row[2] == "":
                status = 0
            else:
                status = int(row[2])
            if status & 1 == 0:
                self.bibdocs.append(BibDoc(bibdocid=row[0],recid=self.id))

    def listBibDocs(self,type=""):
        """Returns the list all bibdocs object belonging to a recid.
        If type is set, it returns just the bibdocs of that type.
        """
        tmp=[]
        for bibdoc in self.bibdocs:
            if type=="" or type == bibdoc.getType():
                tmp.append(bibdoc)
        return tmp

    def getBibDocNames(self,type="Main"):
        """Returns the names of the files associated with the bibdoc of a paritcular type"""
        names = []
        for bibdoc in self.listBibDocs(type):
            names.append(bibdoc.getDocName())
        return names

    def getBibDoc(self,bibdocid):
        """Returns the bibdoc with a particular bibdocid associated with this recid"""
        for bibdoc in self.bibdocs:
            if bibdoc.getId() == bibdocid:
                return bibdoc
        return None

    def deleteBibDoc(self,bibdocid):
        """Delete a bibdocid associated with the recid."""
        for bibdoc in self.bibdocs:
            if bibdoc.getId() == bibdocid:
                bibdoc.delete()
        self.buildBibDocList()

    def addBibDoc(self,type="Main",docname="file"):
        """Creates a new bibdoc associated with the recid, with a file called docname
        and a paritcular type. It returns the bibdoc object which was just created.
        If it already exists a bibdoc with docname name, it appends _2 the necessary
        number of times for having a unique name.
        """
        while docname in self.getBibDocNames(type):
            match = re.match("(.*_)([^_]*)",docname)
            if match:
                try:
                    docname = match.group(1)+str(int(match.group(2)) + 1)
                except:
                    docname = docname + "_2"
            else:
                docname = docname + "_2"
        bibdoc = BibDoc(recid=self.id,type=type,docname=docname)
        if bibdoc is not None:
            self.bibdocs.append(bibdoc)
        return bibdoc

    def addNewFile(self,fullpath,type="Main"):
        """Creates a new bibdoc given a fullpath file and store the file in it.
        It returns the bibdoc object.
        """
        filename = re.sub("\..*","",re.sub(r".*[\\/:]", "", fullpath))
        bibdoc = self.addBibDoc(type,filename)
        if bibdoc is not None:
            bibdoc.addFilesNewVersion(files=[fullpath])
            return bibdoc
        return None

    def addNewVersion(self,fullpath,bibdocid):
        """Adds a new fullpath file to an already existent bibdocid making the previous
        files associated with the same bibdocids obsolete.
        It returns the bibdoc object.
        """
        bibdoc = self.getBibDoc(bibdocid)
        if bibdoc is not None:
            bibdoc.addFilesNewVersion(files=[fullpath])
            docname = re.sub("\..*","",re.sub(r".*[\\/:]", "", fullpath))
            if docname != bibdoc.getDocName():
                while docname in self.getBibDocNames(bibdoc.getType()):
                    match = re.match("(.*_)([^_]*)",docname)
                    if match:
                        try:
                            docname = match.group(1)+str(int(match.group(2)) + 1)
                        except:
                            docname = docname + "_2"
                    else:
                        docname = docname + "_2"
                bibdoc.changeName(docname)
            return bibdoc
        return None

    def addNewFormat(self,fullpath,bibdocid):
        """Adds a new format for a fullpath file to an already existent bibdocid
        along side already there files.
        It returns the bibdoc object.
        """
        bibdoc = self.getBibDoc(bibdocid)
        if bibdoc is not None:
            bibdoc.addFilesNewFormat(files=[fullpath])
            return bibdoc
        return None

    def listLatestFiles(self,type=""):
        docfiles = []
        for bibdoc in self.listBibDocs(type):
            for docfile in bibdoc.listLatestFiles():
                docfiles.append(docfile)
        return docfiles

    def checkFileExists(self,fullpath,type=""):
        if os.path.exists(fullpath):
            docfiles = self.listLatestFiles(type)
            for docfile in docfiles:
                if md5.new(readfile(fullpath)).digest() == md5.new(readfile(docfile.getPath())).digest():
                    return docfile.getBibDocId()
        else:
            return 0


    def display(self,bibdocid="",version="",type="", ln = cdslang):
        t=""
        bibdocs = []
        if bibdocid!="":
            for bibdoc in self.bibdocs:
                if bibdoc.getId() == bibdocid:
                    bibdocs.append(bibdoc)
        else:
            bibdocs = self.listBibDocs(type)
        if len(bibdocs) > 0:
            types = listTypesFromArray(bibdocs)
            fulltypes = []
            for mytype in types:
{'fraction_non_alphanumeric': 0.09791543368185704, 'fraction_numerical': 0.008548694680081542, 'mean_word_length': 2.1093845839296668, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7633391', 'n_tokens_mistral': 4729, 'n_tokens_neox': 4357, 'n_words': 1294}