<commit_before>from . import element, function, util, numpy, parallel, _

class Topology( set ):
  'topology base class'

  def __add__( self, other ):
    'add topologies'

    if self is other:
      return self
    assert self.ndims == other.ndims
    return UnstructuredTopology( set(self) | set(other), ndims=self.ndims )

  def __getitem__( self, item ):
    'subtopology'

    items = ( self.groups[it] for it in item.split( ',' ) )
    return sum( items, items.next() )

  def integrate( self, func, ischeme=None, coords=None, title=True, dense=None ):
    'integrate'

    funcs = func if isinstance( func, (list,tuple) ) else [func]
    shapes = [ map(int,f.shape) for f in funcs ]

    if self.ndims == 0:
      # simple point evaluation
      iweights = 1
      ischeme = 'none'
    else:
      assert coords, 'must specify coords'
      assert ischeme, 'must specify integration scheme'
      iweights = coords.iweights( self.ndims )
    idata = function.Tuple([ function.Tuple([ f, f.indices(), iweights ]) for f in funcs ])

    if not dense and ( isinstance( func, tuple ) or isinstance( func, function.ArrayFunc ) and len( func.shape ) == 2 ):
      # quickly implemented single array for now, needs to be extended for
      # multiple inputs. requires thinking of separating types for separate
      # arguments.

      topo = self if not title \
        else util.progressbar( self, title='integrating %d elements sparse' % len(self) if title is True else title )

      import scipy.sparse

      assert all( sh == shapes[0] for sh in shapes[1:] )

      indices = []
      values = []
      for elem in topo:
        for data, index, w in idata( elem.eval(ischeme) ):
          evalues = util.contract( data, w )
          ndims = evalues.ndim
          eindices = numpy.empty( (ndims,) + evalues.shape )
          for i, n in enumerate( index ):
            eindices[i] = n
          values.append( evalues.ravel() )
          indices.append( eindices.reshape(ndims,-1) )

      v = numpy.hstack( values )
      ij = numpy.hstack( indices )
      A = scipy.sparse.csr_matrix( (v,ij), shape=shapes[0] )

    else:

      topo = self if not title \
        else util.progressbar( self, title='integrating %d elems dense nproc=%d' % ( len(self), parallel.nprocs ) if title is True else title )

      A = map( parallel.shzeros, shapes )

      lock = parallel.Lock()
      for elem in parallel.pariter( topo ):
        for i, (data,index,w) in enumerate( idata( elem.eval(ischeme) ) ):
          emat = util.contract( data, w )
          with lock:<commit_msg>hstack problem fixed<commit_after>from . import element, function, util, numpy, parallel, _

class Topology( set ):
  'topology base class'

  def __add__( self, other ):
    'add topologies'

    if self is other:
      return self
    assert self.ndims == other.ndims
    return UnstructuredTopology( set(self) | set(other), ndims=self.ndims )

  def __getitem__( self, item ):
    'subtopology'

    items = ( self.groups[it] for it in item.split( ',' ) )
    return sum( items, items.next() )

  def integrate( self, func, ischeme=None, coords=None, title=True, dense=None ):
    'integrate'

    funcs = func if isinstance( func, (list,tuple) ) else [func]
    shapes = [ map(int,f.shape) for f in funcs ]

    if self.ndims == 0:
      # simple point evaluation
      iweights = 1
      ischeme = 'none'
    else:
      assert coords, 'must specify coords'
      assert ischeme, 'must specify integration scheme'
      iweights = coords.iweights( self.ndims )
    idata = function.Tuple([ function.Tuple([ f, f.indices(), iweights ]) for f in funcs ])

    if not dense and ( isinstance( func, tuple ) or isinstance( func, function.ArrayFunc ) and len( func.shape ) == 2 ):
      # quickly implemented single array for now, needs to be extended for
      # multiple inputs. requires thinking of separating types for separate
      # arguments.

      topo = self if not title \
        else util.progressbar( self, title='integrating %d elements sparse' % len(self) if title is True else title )

      import scipy.sparse

      assert all( sh == shapes[0] for sh in shapes[1:] )

      indices = []
      values = []
      length = 0
      for elem in topo:
        for data, index, w in idata( elem.eval(ischeme) ):
          evalues = util.contract( data, w )
          ndims = evalues.ndim
          eindices = numpy.empty( (ndims,) + evalues.shape )
          for i, n in enumerate( index ):
            eindices[i] = n
          values.append( evalues.ravel() )
          indices.append( eindices.reshape(ndims,-1) )
          length += evalues.size

      print 'length=%d' % length
      v = numpy.empty( length, dtype=float )
      ij = numpy.empty( [2,length], dtype=float )
      n0 = 0
      for val, ind in zip( values, indices ):
        n1 = n0 + val.size
        v[n0:n1] = val
        ij[:,n0:n1] = ind
        n0 = n1
      assert n0 == length
      A = scipy.sparse.csr_matrix( (v,ij), shape=shapes[0] )

    else:

      topo = self if not title \
        else util.progressbar( self, title='integrating %d elems dense nproc=%d' % ( len(self), parallel.nprocs ) if title is True else title )

      A = map( parallel.shzeros, shapes )

      lock = parallel.Lock()
      for elem in parallel.pariter( topo ):
        for i, (data,index,w) in enumerate( idata( elem.eval(ischeme) ) ):
          emat = util.contract( data, w )
          with lock:
{'fraction_non_alphanumeric': 0.11482024944974321, 'fraction_numerical': 0.004952311078503301, 'mean_word_length': 2.4512658227848103, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2912394', 'n_tokens_mistral': 1741, 'n_tokens_neox': 1680, 'n_words': 555}