<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import csv
import sys
from apiclient.http import MediaFileUpload
import apiclient.errors
import urllib
import requests
import json
import pprint
import re

import logging
logger = logging.getLogger("gdput")
#logger.setLevel(logging.ERROR)

import random
import os

import json

from gdcmdtools.base import GDBase
from gdcmdtools.perm import GDPerm
from gdcmdtools.auth import GDAuth

DICT_OF_CONVERTIBLE_FILE_TYPE = { \
        'raw':[
            "Raw file",
            []],
        'ss':[
            "Spreadsheet",
            ['xls', 'xlsx', 'ods', 'csv', 'tsv', 'tab']],
        'ft':[
            "Fusion Table",
            ['csv']],
        'pt':[
            "Presentation",
            ['ppt', 'pps', 'pptx']],
        'dr':[
            "Drawing",
            ['wmf']],
        'ocr':[
            "OCR",
            ['jpg', 'git', 'png', 'pdf']],
        'doc':[
            "Document",
            ['doc', 'docx', 'html', 'htm', 'txt', 'rtf']],
        'gas':[
            "GAS project",
            ['json']],
        }


# FIXME: naming
class GDPut:
    def __init__(self, **kwargs):

        logger.debug(kwargs)

        for key, value in kwargs.items():
            setattr(self, key, value)

        self.file_id = None
        if self.replace_id:
            file_id_from_link = re.search("^.*/d/([\w\-]*)", self.replace_id)
            
            if file_id_from_link == None:
                self.file_id = self.replace_id
            else:
                self.file_id = file_id_from_link.group(1)

        self.ft_headers = None

        self.csv_latlng_suffix = "_latlng_%04x.csv" % random.getrandbits(16)

        # base
        auth = GDAuth()
        creds = auth.get_credentials()
        self.auth_user = creds.id_token.get("email",None)
        if creds == None:
            raise Exception("Failed to retrieve credentials")
        self.http = auth.get_authorized_http()

        base = GDBase()
        self.service = base.get_drive_service(self.http)
        self.root = base.get_root()

        # ft service
        if self.target_type == "ft":
            self.ft_service = base.get_ft_service(self.http)
 

    def if_folder_writable(self):
        try:
            permissions = self.service.permissions().list(fileId=self.folder_id).execute()
            valid_roles = ["writer", "owner"]
            logger.debug(pprint.pformat(permissions))
            for p in permissions["items"]:
                email = p.get("emailAddress",None).lower()
                role = p.get("role",None).lower()

                logger.debug("email: %s, role: %s" % (email, role))

                if( email == self.auth_user ) and (role in valid_roles):
                    return True
        except:
            return False

        return False
        
    def run(self):
        # check folder_id
        if self.folder_id:
            if self.if_folder_writable() == False:
                raise Exception("folder_id doesn't exist or insufficient permission: %s" % self.folder_id)

        try:
            result = getattr(self, self.target_type+"_put")()
        except AttributeError as e:
            logger.error(e)
            raise 
        except Exception, e:
            logger.error(e)
            raise
            
        return result

    def get_current_user(self):
        pass

    def raw_put(self):
        return self.generic_put(False)

    def check_gas(self):
        # have "id",
        # have at least one file
        # the file should have type, id, name items.
        with open(self.source_file, "rb") as f:
            jsons = json.loads(f.read())
            
            if_ok = False

            if type(jsons) != dict:
                return False
            
            self.file_id = jsons["id"]
            if jsons["id"] and (len(jsons["files"]) > 0):
                for j in jsons["files"]:
                    if j["type"] and j["id"] and j["name"]:
                        if_ok = True
                    else:
                        return False
                        
        return if_ok 

    def gas_pack(self):
        map_type_ext = {"server_js":"js", "html":"html"}
        json_packed = {}
        try:
            with open(self.source_file, "rb") as fr1:
                jsons = json.loads(fr1.read())
                
                path = os.path.split(self.source_file)[0]
                for j in jsons["files"]:
                    file_name = os.path.join(path, "%s.%s" % (j["name"], map_type_ext[j["type"]]))
                    with open(file_name, "rb") as fr2:
                        file_content = fr2.read()

                    j["source"] = file_content

                new_json = "%s.packed" % self.source_file
                with open(new_json, "wb+") as fw:
                    fw.write(json.dumps(jsons, indent=4))
        except:<commit_msg>check id from gas project<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import csv
import sys
from apiclient.http import MediaFileUpload
import apiclient.errors
import urllib
import requests
import json
import pprint
import re

import logging
logger = logging.getLogger("gdput")
#logger.setLevel(logging.ERROR)

import random
import os

import json

from gdcmdtools.base import GDBase
from gdcmdtools.perm import GDPerm
from gdcmdtools.auth import GDAuth

DICT_OF_CONVERTIBLE_FILE_TYPE = { \
        'raw':[
            "Raw file",
            []],
        'ss':[
            "Spreadsheet",
            ['xls', 'xlsx', 'ods', 'csv', 'tsv', 'tab']],
        'ft':[
            "Fusion Table",
            ['csv']],
        'pt':[
            "Presentation",
            ['ppt', 'pps', 'pptx']],
        'dr':[
            "Drawing",
            ['wmf']],
        'ocr':[
            "OCR",
            ['jpg', 'git', 'png', 'pdf']],
        'doc':[
            "Document",
            ['doc', 'docx', 'html', 'htm', 'txt', 'rtf']],
        'gas':[
            "GAS project",
            ['json']],
        }


# FIXME: naming
class GDPut:
    def __init__(self, **kwargs):

        logger.debug(kwargs)

        for key, value in kwargs.items():
            setattr(self, key, value)

        self.file_id = None
        if self.replace_id:
            file_id_from_link = re.search("^.*/d/([\w\-]*)", self.replace_id)
            
            if file_id_from_link == None:
                self.file_id = self.replace_id
            else:
                self.file_id = file_id_from_link.group(1)

        self.ft_headers = None

        self.csv_latlng_suffix = "_latlng_%04x.csv" % random.getrandbits(16)

        # base
        auth = GDAuth()
        creds = auth.get_credentials()
        self.auth_user = creds.id_token.get("email",None)
        if creds == None:
            raise Exception("Failed to retrieve credentials")
        self.http = auth.get_authorized_http()

        base = GDBase()
        self.service = base.get_drive_service(self.http)
        self.root = base.get_root()

        # ft service
        if self.target_type == "ft":
            self.ft_service = base.get_ft_service(self.http)
 

    def if_folder_writable(self):
        try:
            permissions = self.service.permissions().list(fileId=self.folder_id).execute()
            valid_roles = ["writer", "owner"]
            logger.debug(pprint.pformat(permissions))
            for p in permissions["items"]:
                email = p.get("emailAddress",None).lower()
                role = p.get("role",None).lower()

                logger.debug("email: %s, role: %s" % (email, role))

                if( email == self.auth_user ) and (role in valid_roles):
                    return True
        except:
            return False

        return False
        
    def run(self):
        # check folder_id
        if self.folder_id:
            if self.if_folder_writable() == False:
                raise Exception("folder_id doesn't exist or insufficient permission: %s" % self.folder_id)

        try:
            result = getattr(self, self.target_type+"_put")()
        except AttributeError as e:
            logger.error(e)
            raise 
        except Exception, e:
            logger.error(e)
            raise
            
        return result

    def get_current_user(self):
        pass

    def raw_put(self):
        return self.generic_put(False)

    def check_gas(self):
        # have "id",
        # have at least one file
        # the file should have type, id, name items.
        with open(self.source_file, "rb") as f:
            jsons = json.loads(f.read())
            
            if_ok = False

            if type(jsons) != dict:
                return False
            
            if not self.gas_new:
                assert(jsons.get("id"))
                self.file_id = jsons.get("id")
           
            if(len(jsons["files"]) > 0):
                for j in jsons["files"]:
                    if j["type"] and j["name"]:
                        if_ok = True
                    else:
                        return False
                        
        return if_ok 

    def gas_pack(self):
        map_type_ext = {"server_js":"js", "html":"html"}
        json_packed = {}
        try:
            with open(self.source_file, "rb") as fr1:
                jsons = json.loads(fr1.read())
                
                path = os.path.split(self.source_file)[0]
                for j in jsons["files"]:
                    file_name = os.path.join(path, "%s.%s" % (j["name"], map_type_ext[j["type"]]))
                    with open(file_name, "rb") as fr2:
                        file_content = fr2.read()

                    j["source"] = file_content

                new_json = "%s.packed" % self.source_file
                with open(new_json, "wb+") as fw:
                    fw.write(json.dumps(jsons, indent=4))
        except:
{'fraction_non_alphanumeric': 0.12560483870967742, 'fraction_numerical': 0.002620967741935484, 'mean_word_length': 1.5497301464919044, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4373899', 'n_tokens_mistral': 3053, 'n_tokens_neox': 2930, 'n_words': 670}