<commit_before>#!/usr/bin/env python

import zipfile
import os
import json
import sys
import time
import shutil
import hashlib
import datetime
import pyclamd
import ConfigParser
import httplib
import urllib2
from BeautifulSoup              import BeautifulSoup as bsoup
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy                 import Table, Column, Integer, String, \
                                       DateTime, Date, ForeignKey, Text, \
                                       Boolean, create_engine, MetaData, and_
from sqlalchemy.orm             import relation, backref, sessionmaker


def _config(stanza, option, opt_type='string'):
  '''
  Open the configuration file and pull the value fromt he stanza and option
  specified.  Optionally the option type can be overriden with opt_type to
  return any of the following types: string, bool, float, and int.
  '''
  config = ConfigParser.ConfigParser()
  config.read('config.ini')
  if opt_type == 'string':
    return config.get(stanza, option)
  if opt_type == 'int':
    return config.getint(stanza, option)
  if opt_type == 'bool':
    return config.getboolean(stanza, option)
  if opt_type == 'float':
    return config.getfloat(stanza, option)
  if opt_type == 'path':
    return os.path.normpath(config.get(stanza, option))

# Open the port to the Clamd daemon.  this will be needed to scan the
# packages as they are initialized to make sure that therte arent any viruses
# in the packages.
pyclamd.init_network_socket(_config('Settings', 'clamd_host'), 
                            _config('Settings', 'clamd_port', 'int'))

sql_string    = _config('Settings', 'database')
engine        = create_engine(sql_string)
Session       = sessionmaker(bind=engine)
Base          = declarative_base()

def get_session():
  session       = Session()
  return session

class UserAPI(Base):
  '''
  This is the class supporting the user API keys that are stored in the
  database.  This class will also handle requesting a new key for a user as
  well.
  '''
  __tablename__ = 'users'
  api           = Column(String(255), primary_key=True)
  username      = Column(String(255), primary_key=True)
  email         = Column(String(255))
  activated     = Column(Boolean)
  
  def __init__(self, name, email):
    self.username   = name
    self.email      = email
    md5             = hashlib.md5()
    md5.update(name + email + datetime.datetime.now().ctime())
    self.api        = md5.hexdigest()
    self.activated  = False
    
class BukGetPkg(Base):
  '''
  This is the generic bukget package object.  Everything from verification,
  addition into the repository, and dumping individual bukget-compliant
  dictionaries are handled within this object.
  '''
  __tablename__ = 'packages'
  dictionary    = None
  status        = None
  valid         = None
  _base         = None
  _repo         = None
  filename      = None
  new_filename  = None
  _engine       = None
  _session      = None
  name          = Column(String(64), primary_key=True)
  version       = Column(String(15), primary_key=True)
  branch        = Column(String(10))
  pkg_type      = Column(String(10))
  author        = Column(String(255))
  description   = Column(Text)
  website       = Column(String(255))
  location      = Column(String(255))
  checksum      = Column(String(255))
  categories    = Column(Text)
  bukkit_min    = Column(Integer)
  bukkit_max    = Column(Integer)
  required_deps = Column(Text)
  optional_deps = Column(Text)
  
  def get_deps(self):
    '''
    Returns the dependency dictionary.
    '''
    return {
      'required': self.required_deps.split(','),
      'optional': self.optional_deps.split(','),
    }
  
  def get_cats(self):
    '''
    Returns the category list.
    '''
    return self.categories.split(',')

  def set_deps(self, values):
    '''
    Reformats the dependency dictionary to be included in the database.
    '''
    self.required_deps  = ','.join(values['required'])
    self.optional_deps  = ','.join(values['optional'])
  
  def set_cats(self, values):
    '''
    Reformats the category list to be included int he database.
    '''
    self.categories     = ','.join(values)
  
  def __init__(self, filename):
    '''
    Initialize and validate the package that has been 
    '''
    
    # First thing we need to pull in the information we need from the config
    # file and set the filename to be persistant in the object.
    self._base          = _config('Settings', 'urlbase')
    self._repo          = _config('Settings', 'repository', 'path')
    self.filename       = os.path.join(_config('Settings', 'uploads'), filename)
    
    # Next we will perform a quick scan of the zipfile to make sure that it
    # is clean of any malware that we are able to detect.
    avstatus            = pyclamd.scan_file(self.filename)
    if avstatus is not None:
      self.valid        = False
      self.status       = 'ClamAV: %s' % avstatus
      return
    
    # Now we want to make sure that the file is really a zipfile. Otherwise
    # there is no point on trying to continue.
    if not zipfile.is_zipfile(self.filename):
      self.valid        = False
      self.status       = 'Not a Zip File'
      return
    
    # Lets go ahead and generate a ZipFile object with the package file so
    # we can start parsing. :)
    package             = zipfile.ZipFile(self.filename, 'a')
    
    # Next we need to make sure that there is at leats a basic structure in
    # the package and that there is at least 1 jar file in the package in
    # order for this to be a valid package.
    checks = 0
    for fname in package.filelist:
      if fname.filename[:3] in ['bin', 'etc', 'lib']:
        checks += 1
      if fname.filename == 'info.json' or fname.filename[-3:] == 'jar':
        checks += 1
    if checks < 5:
      self.valid        = False
      self.status       = 'Not in a Valid Package Format'
      return
    
    # And now we will start to import the info.json file into the object so
    # that we can start using the information later on if we want to add the<commit_msg>Swicthed to using clamscan from pyclamd.<commit_after>#!/usr/bin/env python

import zipfile
import os
import json
import sys
import time
import shutil
import hashlib
import datetime
#import pyclamd
import ConfigParser
import httplib
import urllib2
from commands                   import getstatusoutput  as run
from BeautifulSoup              import BeautifulSoup    as bsoup
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy                 import Table, Column, Integer, String, \
                                       DateTime, Date, ForeignKey, Text, \
                                       Boolean, create_engine, MetaData, and_
from sqlalchemy.orm             import relation, backref, sessionmaker


def _config(stanza, option, opt_type='string'):
  '''
  Open the configuration file and pull the value fromt he stanza and option
  specified.  Optionally the option type can be overriden with opt_type to
  return any of the following types: string, bool, float, and int.
  '''
  config = ConfigParser.ConfigParser()
  config.read('config.ini')
  if opt_type == 'string':
    return config.get(stanza, option)
  if opt_type == 'int':
    return config.getint(stanza, option)
  if opt_type == 'bool':
    return config.getboolean(stanza, option)
  if opt_type == 'float':
    return config.getfloat(stanza, option)
  if opt_type == 'path':
    return os.path.normpath(config.get(stanza, option))

# Open the port to the Clamd daemon.  this will be needed to scan the
# packages as they are initialized to make sure that therte arent any viruses
# in the packages.
#pyclamd.init_network_socket(_config('Settings', 'clamd_host'), 
#                            _config('Settings', 'clamd_port', 'int'))

sql_string    = _config('Settings', 'database')
engine        = create_engine(sql_string)
Session       = sessionmaker(bind=engine)
Base          = declarative_base()

def get_session():
  session       = Session()
  return session

class UserAPI(Base):
  '''
  This is the class supporting the user API keys that are stored in the
  database.  This class will also handle requesting a new key for a user as
  well.
  '''
  __tablename__ = 'users'
  api           = Column(String(255), primary_key=True)
  username      = Column(String(255), primary_key=True)
  email         = Column(String(255))
  activated     = Column(Boolean)
  
  def __init__(self, name, email):
    self.username   = name
    self.email      = email
    md5             = hashlib.md5()
    md5.update(name + email + datetime.datetime.now().ctime())
    self.api        = md5.hexdigest()
    self.activated  = False
    
class BukGetPkg(Base):
  '''
  This is the generic bukget package object.  Everything from verification,
  addition into the repository, and dumping individual bukget-compliant
  dictionaries are handled within this object.
  '''
  __tablename__ = 'packages'
  dictionary    = None
  status        = None
  valid         = None
  _base         = None
  _repo         = None
  filename      = None
  new_filename  = None
  _engine       = None
  _session      = None
  name          = Column(String(64), primary_key=True)
  version       = Column(String(15), primary_key=True)
  branch        = Column(String(10))
  pkg_type      = Column(String(10))
  author        = Column(String(255))
  description   = Column(Text)
  website       = Column(String(255))
  location      = Column(String(255))
  checksum      = Column(String(255))
  categories    = Column(Text)
  bukkit_min    = Column(Integer)
  bukkit_max    = Column(Integer)
  required_deps = Column(Text)
  optional_deps = Column(Text)
  
  def get_deps(self):
    '''
    Returns the dependency dictionary.
    '''
    return {
      'required': self.required_deps.split(','),
      'optional': self.optional_deps.split(','),
    }
  
  def get_cats(self):
    '''
    Returns the category list.
    '''
    return self.categories.split(',')

  def set_deps(self, values):
    '''
    Reformats the dependency dictionary to be included in the database.
    '''
    self.required_deps  = ','.join(values['required'])
    self.optional_deps  = ','.join(values['optional'])
  
  def set_cats(self, values):
    '''
    Reformats the category list to be included int he database.
    '''
    self.categories     = ','.join(values)
  
  def __init__(self, filename):
    '''
    Initialize and validate the package that has been 
    '''
    
    # First thing we need to pull in the information we need from the config
    # file and set the filename to be persistant in the object.
    self._base          = _config('Settings', 'urlbase')
    self._repo          = _config('Settings', 'repository', 'path')
    self.filename       = os.path.join(_config('Settings', 'uploads'), filename)
    
    # Next we will perform a quick scan of the zipfile to make sure that it
    # is clean of any malware that we are able to detect.
    avstatus            = run('clamscan %s' % self.filename)
    if avstatus[0] > 0:
      self.valid        = False
      self.status       = 'ClamAV reported a possible infection.'
      return
    
    # Now we want to make sure that the file is really a zipfile. Otherwise
    # there is no point on trying to continue.
    if not zipfile.is_zipfile(self.filename):
      self.valid        = False
      self.status       = 'Not a Zip File'
      return
    
    # Lets go ahead and generate a ZipFile object with the package file so
    # we can start parsing. :)
    package             = zipfile.ZipFile(self.filename, 'a')
    
    # Next we need to make sure that there is at leats a basic structure in
    # the package and that there is at least 1 jar file in the package in
    # order for this to be a valid package.
    checks = 0
    for fname in package.filelist:
      if fname.filename[:3] in ['bin', 'etc', 'lib']:
        checks += 1
      if fname.filename == 'info.json' or fname.filename[-3:] == 'jar':
        checks += 1
    if checks < 5:
      self.valid        = False
      self.status       = 'Not in a Valid Package Format'
      return
    
    # And now we will start to import the info.json file into the object so
    # that we can start using the information later on if we want to add the
{'fraction_non_alphanumeric': 0.09175499635892871, 'fraction_numerical': 0.006796666396957683, 'mean_word_length': 2.537492844876932, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2707273', 'n_tokens_mistral': 3655, 'n_tokens_neox': 3551, 'n_words': 1228}