<commit_before>#!/usr/bin/env python

# Copyright (C) 2009-2014:
#    <NAME>, <EMAIL>
#    <NAME>, <EMAIL>
#    <NAME>, <EMAIL>
#    <NAME>, <EMAIL>
#
# This file is part of Shinken.
#
# Shinken is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Shinken is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Shinken.  If not, see <http://www.gnu.org/licenses/>.


import pycurl
import os
import sys
import stat
import json
import tempfile
import tarfile
import urllib
import shutil
from StringIO import StringIO
from shinken.log import logger, cprint

# Will be populated by the shinken CLI command
CONFIG = None



############# ********************        PUBLISH           ****************###########

def read_package_json(fd):
    buf = fd.read()
    fd.close()
    buf = buf.decode('utf8', 'ignore')
    try:
        package_json = json.loads(buf)
    except ValueError, exp:
        logger.error("Bad package.json file : %s" % exp)
        sys.exit(2)
    if not package_json:
        logger.error("Bad package.json file")
        sys.exit(2)
    return package_json




def create_archive(to_pack):
    # First try to look if the directory we are trying to pack is valid
    to_pack = os.path.abspath(to_pack)
    if not os.path.exists(to_pack):
        logger.error("Error : the directory to pack is missing %s" % to_pack)
        sys.exit(2)
    logger.debug("Preparing to pack the directory %s" % to_pack)
    package_json_p = os.path.join(to_pack, 'package.json')
    if not os.path.exists(package_json_p):
        logger.error("Error : Missing file %s" % package_json_p)
        sys.exit(2)
    package_json = read_package_json(open(package_json_p))

    name = package_json.get('name', None)
    if not name:
        logger.error('Missing name entry in the package.json file. Cannot pack')
        sys.exit(2)


    # return True for files we want to exclude
    def tar_exclude_filter(f):
        # if the file start with .git, we bail out
        # Also ending with ~ (Thanks emacs...)
        if f.startswith('./.git') or f.startswith('.git'):
            return True
        if f.endswith('~'):
            return True
        return False

    # Now prepare a destination file
    tmp_dir  = tempfile.gettempdir()
    tmp_file = os.path.join(tmp_dir, name+'.tar.gz')
    tar = tarfile.open(tmp_file, "w:gz")
    os.chdir(to_pack)
    tar.add(".",arcname='.', exclude=tar_exclude_filter)
    tar.close()
    logger.debug("Saved file %s" % tmp_file)
    return tmp_file


def publish_archive(archive):
    # Now really publish it
    proxy = CONFIG['shinken.io']['proxy']
    api_key = CONFIG['shinken.io']['api_key']

    # Ok we will push the file with a 10s timeout
    c = pycurl.Curl()
    c.setopt(c.POST, 1)
    c.setopt(c.CONNECTTIMEOUT, 30)
    c.setopt(c.TIMEOUT, 300)
    if proxy:
        c.setopt(c.PROXY, proxy)
    c.setopt(c.URL, "http://shinken.io/push")
    c.setopt(c.HTTPPOST, [("api_key", api_key),
                          ("data",
                           (c.FORM_FILE, str(archive),
                            c.FORM_CONTENTTYPE, "application/x-gzip"))
                          ])
    response = StringIO()
    c.setopt(pycurl.WRITEFUNCTION, response.write)
    c.setopt(c.VERBOSE, 1)
    try:
        c.perform()
    except pycurl.error, exp:
        logger.error("There was a critical error : %s" % exp)
        return
    r = c.getinfo(pycurl.HTTP_CODE)
    c.close()
    if r != 200:
        logger.error("There was a critical error : %s" % response.getvalue())
        sys.exit(2)
    else:
        ret  = json.loads(response.getvalue().replace('\\/', '/'))
        status = ret.get('status')
        text   = ret.get('text')
        if status == 200:
            logger.log(text)
        else:
            logger.error(text)


def do_publish(to_pack='.'):
    logger.debug("WILL CALL PUBLISH.py with %s" % to_pack)
    archive = create_archive(to_pack)
    publish_archive(archive)




################" *********************** SEARCH *************** ##################
def search(look_at):
    # Now really publish it
    proxy = CONFIG['shinken.io']['proxy']
    api_key = CONFIG['shinken.io']['api_key']

    # Ok we will push the file with a 10s timeout
    c = pycurl.Curl()
    c.setopt(c.POST, 0)
    c.setopt(c.CONNECTTIMEOUT, 30)
    c.setopt(c.TIMEOUT, 300)
    if proxy:
        c.setopt(c.PROXY, proxy)

    args = {'keywords':','.join(look_at)}
    c.setopt(c.URL, str('shinken.io/searchcli?'+urllib.urlencode(args)))
    response = StringIO()
    c.setopt(pycurl.WRITEFUNCTION, response.write)
    #c.setopt(c.VERBOSE, 1)
    try:
        c.perform()
    except pycurl.error, exp:
        logger.error("There was a critical error : %s" % exp)
        return

    r = c.getinfo(pycurl.HTTP_CODE)
    c.close()
    if r != 200:
        logger.error("There was a critical error : %s" % response.getvalue())
        sys.exit(2)
    else:
        ret  = json.loads(response.getvalue().replace('\\/', '/'))
        status = ret.get('status')
        result   = ret.get('result')
        if status != 200:
            logger.log(result)
            return []
        return result<commit_msg>Replaced logger.log calls with logger.info<commit_after>#!/usr/bin/env python

# Copyright (C) 2009-2014:
#    <NAME>, <EMAIL>
#    <NAME>, <EMAIL>
#    <NAME>, <EMAIL>
#    <NAME>, <EMAIL>
#
# This file is part of Shinken.
#
# Shinken is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Shinken is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Shinken.  If not, see <http://www.gnu.org/licenses/>.


import pycurl
import os
import sys
import stat
import json
import tempfile
import tarfile
import urllib
import shutil
from StringIO import StringIO
from shinken.log import logger, cprint

# Will be populated by the shinken CLI command
CONFIG = None



############# ********************        PUBLISH           ****************###########

def read_package_json(fd):
    buf = fd.read()
    fd.close()
    buf = buf.decode('utf8', 'ignore')
    try:
        package_json = json.loads(buf)
    except ValueError, exp:
        logger.error("Bad package.json file : %s" % exp)
        sys.exit(2)
    if not package_json:
        logger.error("Bad package.json file")
        sys.exit(2)
    return package_json




def create_archive(to_pack):
    # First try to look if the directory we are trying to pack is valid
    to_pack = os.path.abspath(to_pack)
    if not os.path.exists(to_pack):
        logger.error("Error : the directory to pack is missing %s" % to_pack)
        sys.exit(2)
    logger.debug("Preparing to pack the directory %s" % to_pack)
    package_json_p = os.path.join(to_pack, 'package.json')
    if not os.path.exists(package_json_p):
        logger.error("Error : Missing file %s" % package_json_p)
        sys.exit(2)
    package_json = read_package_json(open(package_json_p))

    name = package_json.get('name', None)
    if not name:
        logger.error('Missing name entry in the package.json file. Cannot pack')
        sys.exit(2)


    # return True for files we want to exclude
    def tar_exclude_filter(f):
        # if the file start with .git, we bail out
        # Also ending with ~ (Thanks emacs...)
        if f.startswith('./.git') or f.startswith('.git'):
            return True
        if f.endswith('~'):
            return True
        return False

    # Now prepare a destination file
    tmp_dir  = tempfile.gettempdir()
    tmp_file = os.path.join(tmp_dir, name+'.tar.gz')
    tar = tarfile.open(tmp_file, "w:gz")
    os.chdir(to_pack)
    tar.add(".",arcname='.', exclude=tar_exclude_filter)
    tar.close()
    logger.debug("Saved file %s" % tmp_file)
    return tmp_file


def publish_archive(archive):
    # Now really publish it
    proxy = CONFIG['shinken.io']['proxy']
    api_key = CONFIG['shinken.io']['api_key']

    # Ok we will push the file with a 10s timeout
    c = pycurl.Curl()
    c.setopt(c.POST, 1)
    c.setopt(c.CONNECTTIMEOUT, 30)
    c.setopt(c.TIMEOUT, 300)
    if proxy:
        c.setopt(c.PROXY, proxy)
    c.setopt(c.URL, "http://shinken.io/push")
    c.setopt(c.HTTPPOST, [("api_key", api_key),
                          ("data",
                           (c.FORM_FILE, str(archive),
                            c.FORM_CONTENTTYPE, "application/x-gzip"))
                          ])
    response = StringIO()
    c.setopt(pycurl.WRITEFUNCTION, response.write)
    c.setopt(c.VERBOSE, 1)
    try:
        c.perform()
    except pycurl.error, exp:
        logger.error("There was a critical error : %s" % exp)
        return
    r = c.getinfo(pycurl.HTTP_CODE)
    c.close()
    if r != 200:
        logger.error("There was a critical error : %s" % response.getvalue())
        sys.exit(2)
    else:
        ret  = json.loads(response.getvalue().replace('\\/', '/'))
        status = ret.get('status')
        text   = ret.get('text')
        if status == 200:
            logger.info(text)
        else:
            logger.error(text)


def do_publish(to_pack='.'):
    logger.debug("WILL CALL PUBLISH.py with %s" % to_pack)
    archive = create_archive(to_pack)
    publish_archive(archive)




################" *********************** SEARCH *************** ##################
def search(look_at):
    # Now really publish it
    proxy = CONFIG['shinken.io']['proxy']
    api_key = CONFIG['shinken.io']['api_key']

    # Ok we will push the file with a 10s timeout
    c = pycurl.Curl()
    c.setopt(c.POST, 0)
    c.setopt(c.CONNECTTIMEOUT, 30)
    c.setopt(c.TIMEOUT, 300)
    if proxy:
        c.setopt(c.PROXY, proxy)

    args = {'keywords':','.join(look_at)}
    c.setopt(c.URL, str('shinken.io/searchcli?'+urllib.urlencode(args)))
    response = StringIO()
    c.setopt(pycurl.WRITEFUNCTION, response.write)
    #c.setopt(c.VERBOSE, 1)
    try:
        c.perform()
    except pycurl.error, exp:
        logger.error("There was a critical error : %s" % exp)
        return

    r = c.getinfo(pycurl.HTTP_CODE)
    c.close()
    if r != 200:
        logger.error("There was a critical error : %s" % response.getvalue())
        sys.exit(2)
    else:
        ret  = json.loads(response.getvalue().replace('\\/', '/'))
        status = ret.get('status')
        result   = ret.get('result')
        if status != 200:
            logger.info(result)
            return []
        return result
{'fraction_non_alphanumeric': 0.14388489208633093, 'fraction_numerical': 0.008348876454392041, 'mean_word_length': 2.562163872192344, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7420002', 'n_tokens_mistral': 3755, 'n_tokens_neox': 3617, 'n_words': 995}