<commit_before>import re
import urllib
import urllib2
import urlparse

from lxml import etree
from lxml.builder import E

from sharepoint.xml import SP, namespaces, OUT
from sharepoint.lists.types import type_mapping, default_type

uuid_re = re.compile(r'^\{?([\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})\}?$')

LIST_WEBSERVICE = '_vti_bin/Lists.asmx'

# From http://msdn.microsoft.com/en-us/library/lists.lists.addlist%28v=office.12%29.aspx
LIST_TEMPLATES = {
    'Announcements': 104,
    'Contacts': 105,
    'Custom List': 100,
    'Custom List in Datasheet View': 120,
    'DataSources': 110,
    'Discussion Board': 108,
    'Document Library': 101,
    'Events': 106,
    'Form Library': 115,
    'Issues': 1100,
    'Links': 103,
    'Picture Library': 109,
    'Survey': 102,
    'Tasks': 107
}

class SharePointLists(object):
    def __init__(self, opener):
        self.opener = opener

    @property
    def all_lists(self):
        if not hasattr(self, '_all_lists'):
            xml = SP.GetListCollection()
            result = self.opener.post_soap(LIST_WEBSERVICE, xml)
    
            self._all_lists = []
            for list_element in result.xpath('sp:GetListCollectionResult/sp:Lists/sp:List', namespaces=namespaces):
                self._all_lists.append(SharePointList(self.opener, self, dict(list_element.attrib)))
        return self._all_lists

    def remove(self, list):
        """
        Removes a list from the site.
        """
        xml = SP.DeleteList(SP.listName(list.id))
        result = self.opener.post_soap(LIST_WEBSERVICE, xml,
                                       soapaction='http://schemas.microsoft.com/sharepoint/soap/DeleteList')
        self.all_lists.remove(list)

    def create(self, name, description='', template=100):
        """
        Creates a new list in the site.
        """
        try:
            template = int(template)
        except ValueError:
            template = LIST_TEMPLATES[template]
        if name in self:
            raise ValueError("List already exists: '{0}".format(name))
        if uuid_re.match(name):
            raise ValueError("Cannot create a list with a UUID as a name")
        xml = SP.AddList(SP.listName(name),
                         SP.description(description),
                         SP.templateID(unicode(template)))
        result = self.opener.post_soap(LIST_WEBSERVICE, xml,
                                       soapaction='http://schemas.microsoft.com/sharepoint/soap/AddList')

    def __iter__(self):
        return iter(self.all_lists)

    def __getitem__(self, key):
        if isinstance(key, int):
            return self.all_lists[key]
        elif uuid_re.match(key.lower()):
            # Using group 1 and adding braces allows us to match IDs that
            # didn't originally have braces.
            key = '{' + '{0}'.format(uuid_re.match(key.lower()).group(1)) + '}'
            for list_object in self.all_lists:
                if list_object.id == key:
                    return list_object
            raise KeyError('No list with ID {0}'.format(key))
        elif isinstance(key, basestring):
            for list_object in self.all_lists:
                if list_object.meta['Title'] == key:
                    return list_object
            raise KeyError("No list with title '{0}'".format(key))
        raise KeyError

    def __contains__(self, key):
        try:
            self[key]
        except KeyError:
            return False
        else:
            return True

    def as_xml(self, keys=None, **kwargs):
        if keys is not None:
            lists = (self[key] for key in keys)
        else:
            lists = self
        return OUT.lists(*[l.as_xml(**kwargs) for l in lists])

class SharePointList(object):
    def __init__(self, opener, lists, meta):
        self.opener = opener
        self.lists, self.meta = lists, meta
        self.id = meta['ID'].lower()
        self._deleted_rows = set()

    def __repr__(self):
        return "<SharePointList {0} '{1}'>".format(self.id, self.meta['Title'])

    @property
    def name(self):
        return self.meta['Title']

    @property
    def settings(self):
        if not hasattr(self, '_settings'):
            xml = SP.GetList(SP.listName(self.id))
            response = self.opener.post_soap(LIST_WEBSERVICE, xml)
            self._settings = response[0][0]
        return self._settings

    @property
    def rows(self):
        if not hasattr(self, '_rows'):
            xml = SP.GetListItems(SP.listName(self.id), SP.rowLimit("100000"))
            response = self.opener.post_soap(LIST_WEBSERVICE, xml)
            xml_rows = list(response[0][0][0])
            self._rows = []
            for xml_row in xml_rows:
                self._rows.append(self.Row(xml_row))
        return list(self._rows)

    @property
    def rows_by_id(self):
        if not hasattr(self, '_rows_by_id'):
            self._rows_by_id = {}
            for row in self.rows:
                self._rows_by_id[row.id] = row
        return self._rows_by_id<commit_msg>Combined meta/settings handling on lists.<commit_after>import re
import urllib
import urllib2
import urlparse

from lxml import etree
from lxml.builder import E

from sharepoint.xml import SP, namespaces, OUT
from sharepoint.lists.types import type_mapping, default_type

uuid_re = re.compile(r'^\{?([\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})\}?$')

LIST_WEBSERVICE = '_vti_bin/Lists.asmx'

# From http://msdn.microsoft.com/en-us/library/lists.lists.addlist%28v=office.12%29.aspx
LIST_TEMPLATES = {
    'Announcements': 104,
    'Contacts': 105,
    'Custom List': 100,
    'Custom List in Datasheet View': 120,
    'DataSources': 110,
    'Discussion Board': 108,
    'Document Library': 101,
    'Events': 106,
    'Form Library': 115,
    'Issues': 1100,
    'Links': 103,
    'Picture Library': 109,
    'Survey': 102,
    'Tasks': 107
}

class SharePointLists(object):
    def __init__(self, opener):
        self.opener = opener

    @property
    def all_lists(self):
        if not hasattr(self, '_all_lists'):
            xml = SP.GetListCollection()
            result = self.opener.post_soap(LIST_WEBSERVICE, xml)
    
            self._all_lists = []
            for list_element in result.xpath('sp:GetListCollectionResult/sp:Lists/sp:List', namespaces=namespaces):
                self._all_lists.append(SharePointList(self.opener, self, list_element))
        return self._all_lists

    def remove(self, list):
        """
        Removes a list from the site.
        """
        xml = SP.DeleteList(SP.listName(list.id))
        result = self.opener.post_soap(LIST_WEBSERVICE, xml,
                                       soapaction='http://schemas.microsoft.com/sharepoint/soap/DeleteList')
        self.all_lists.remove(list)

    def create(self, name, description='', template=100):
        """
        Creates a new list in the site.
        """
        try:
            template = int(template)
        except ValueError:
            template = LIST_TEMPLATES[template]
        if name in self:
            raise ValueError("List already exists: '{0}".format(name))
        if uuid_re.match(name):
            raise ValueError("Cannot create a list with a UUID as a name")
        xml = SP.AddList(SP.listName(name),
                         SP.description(description),
                         SP.templateID(unicode(template)))
        result = self.opener.post_soap(LIST_WEBSERVICE, xml,
                                       soapaction='http://schemas.microsoft.com/sharepoint/soap/AddList')
        list_element = result.xpath('sp:AddListResult/sp:List', namespaces=namespaces)[0]
        self._all_lists.append(SharePointList(self.opener, self, list_element))

    def __iter__(self):
        return iter(self.all_lists)

    def __getitem__(self, key):
        if isinstance(key, int):
            return self.all_lists[key]
        elif uuid_re.match(key.lower()):
            # Using group 1 and adding braces allows us to match IDs that
            # didn't originally have braces.
            key = '{' + '{0}'.format(uuid_re.match(key.lower()).group(1)) + '}'
            for list_object in self.all_lists:
                if list_object.id == key:
                    return list_object
            raise KeyError('No list with ID {0}'.format(key))
        elif isinstance(key, basestring):
            for list_object in self.all_lists:
                if list_object.meta['Title'] == key:
                    return list_object
            raise KeyError("No list with title '{0}'".format(key))
        raise KeyError

    def __contains__(self, key):
        try:
            self[key]
        except KeyError:
            return False
        else:
            return True

    def as_xml(self, keys=None, **kwargs):
        if keys is not None:
            lists = (self[key] for key in keys)
        else:
            lists = self
        return OUT.lists(*[l.as_xml(**kwargs) for l in lists])

class SharePointList(object):
    def __init__(self, opener, lists, settings):
        self.opener = opener
        self.lists = lists
        self._deleted_rows = set()
        self._settings, self._meta = settings, None
        self.id = self.meta['ID'].lower()

    def __repr__(self):
        return "<SharePointList {0} '{1}'>".format(self.id, self.meta['Title'])

    @property
    def name(self):
        return self.meta['Title']

    @property
    def meta(self):
        if not self._meta:
            settings = self._settings if self._settings is not None else self.settings
            self._meta = dict(settings.attrib)
        return self._meta

    @property
    def settings(self):
        if not self._settings or not len(self._settings):
            xml = SP.GetList(SP.listName(self.id))
            response = self.opener.post_soap(LIST_WEBSERVICE, xml)
            self._settings = response[0][0]
        return self._settings

    @property
    def rows(self):
        if not hasattr(self, '_rows'):
            xml = SP.GetListItems(SP.listName(self.id), SP.rowLimit("100000"))
            response = self.opener.post_soap(LIST_WEBSERVICE, xml)
            xml_rows = list(response[0][0][0])
            self._rows = []
            for xml_row in xml_rows:
                self._rows.append(self.Row(xml_row))
        return list(self._rows)

    @property
    def rows_by_id(self):
        if not hasattr(self, '_rows_by_id'):
            self._rows_by_id = {}
            for row in self.rows:
                self._rows_by_id[row.id] = row
        return self._rows_by_id
{'fraction_non_alphanumeric': 0.1197103629866466, 'fraction_numerical': 0.014763964641715253, 'mean_word_length': 2.135318396226415, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 38}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1656798', 'n_tokens_mistral': 3358, 'n_tokens_neox': 3170, 'n_words': 738}