<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2013 <NAME>

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

#
# Copyright (C) 2013 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Command-line skeleton application for Freebase API.
Usage:
  $ python selexsubmitter.py

You can also get help on all the command-line flags the program understands
by running:

  $ python selexsubmitter.py --help

"""

import argparse
import httplib2
import os
import sys
import json
import urllib
import os.path
import re
import RNA
import time

from urllib import urlencode
from apiclient import discovery
from oauth2client import file
from oauth2client import client
from oauth2client import tools

#a dict of target names to mids
target_dict = {}

# Parser for command-line arguments.
parser = argparse.ArgumentParser(
    description=__doc__,
    formatter_class=argparse.RawDescriptionHelpFormatter,
    parents=[tools.argparser])
parser.add_argument('-su', '--servlet_url', help='the URL of the servlet that parses the JSON files', required=True)
parser.add_argument('-if', '--input_file', help='an input JSON file', required=True)

# CLIENT_SECRETS is name of a file containing the OAuth 2.0 information for this
# application, including client_id and client_secret. You can see the Client ID
# and Client secret on the APIs page in the Cloud Console:
# <https://cloud.google.com/console#/project/1083819269171/apiui>
CLIENT_SECRETS = os.path.join(os.path.dirname(__file__), 'client_secrets.json')

# Set up a Flow object to be used for authentication.
# Add one or more of the following scopes. PLEASE ONLY ADD THE SCOPES YOU
# NEED. For more information on using scopes please see
# <https://developers.google.com/+/best-practices>.
FLOW = client.flow_from_clientsecrets(CLIENT_SECRETS,
  scope=[
      'https://www.googleapis.com/auth/freebase',
    ],
    message=tools.message_if_missing(CLIENT_SECRETS))


def main(argv):
  # Parse the command-line flags.
  flags = parser.parse_args(argv[1:])
  #define some vars 
  service_url_write = 'https://www.googleapis.com/freebase/v1sandbox/mqlwrite'
  inputFile = flags.input_file
  servlet_url = flags.servlet_url

  # If the credentials don't exist or are invalid run through the native client
  # flow. The Storage object will ensure that if successful the good
  # credentials will get written back to the file.
  storage = file.Storage('sample.dat')
  credentials = storage.get()
  if credentials is None or credentials.invalid:
    credentials = tools.run_flow(FLOW, storage, flags)

  # Create an httplib2.Http object to handle our HTTP requests and authorize it
  # with our good Credentials.
  http = httplib2.Http()
  http = credentials.authorize(http)
  # Construct the service object for the interacting with the Freebase API.
  service = discovery.build('freebase', 'v1', http=http)
  
  if inputFile:
    print inputFile
    #start = time.time()
    # cleanJson = getCleanJson(servlet_url, inputFilesPath ,fn)
    # if cleanJson:
    #   #now prepare a write query for the cleanJSON
    #   se_mid = writeToFreebase(cleanJson, service_url_write, http, credentials)
    #   print "created selex experiment topic with mid: "+se_mid["mid"]
    #   end = time.time()
    #   tt = end-start
    #   print "time elapsed in seconds: "+str(tt)

def writeToFreebase(cleanJson, aServiceUrl, anHttp, someCredentials):
  #create an empty selex experiment topic and get its mid
  mid_dict = createSelexExperimentTopic(aServiceUrl, anHttp, someCredentials)
  #add the reference details from this experiment
  addReferenceDetails(mid_dict, cleanJson, aServiceUrl, anHttp, someCredentials)
  addSelexDetails(mid_dict, cleanJson, aServiceUrl, anHttp, someCredentials)
  addSelexConditions(mid_dict, cleanJson, aServiceUrl, anHttp,someCredentials)
  addInteractions(mid_dict["mid"], cleanJson, aServiceUrl, anHttp, someCredentials)
  return mid_dict

#Creates a empty interaction topic including and returns the mid
# and connects it to the given selex experiment mid
def createInteractionTopic(aSelexExperimentMid, aServiceUrl, anHttp, someCredentials):
  q = {
    "create":"unconditional",
    "mid":None,
    "type":"/base/aptamer/interaction",
    "b:type":"/base/aptamer/experimental_outcome",
    "/base/aptamer/experimental_outcome/is_outcome_of":{
      "connect":"insert",
      "mid" : aSelexExperimentMid
    }
  }
  params = makeRequestBody(someCredentials, q)
  r = runQuery(params, aServiceUrl, anHttp)
  if r == None:
    raise Exception ("Could not create interaction topic!")
    sys.exit()
  return r
<commit_msg>finished fix to make work on one file at a time<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2013 <NAME>

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

#
# Copyright (C) 2013 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Command-line skeleton application for Freebase API.
Usage:
  $ python selexsubmitter.py

You can also get help on all the command-line flags the program understands
by running:

  $ python selexsubmitter.py --help

"""

import argparse
import httplib2
import os
import sys
import json
import urllib
import os.path
import re
import RNA
import time

from urllib import urlencode
from apiclient import discovery
from oauth2client import file
from oauth2client import client
from oauth2client import tools

#a dict of target names to mids
target_dict = {}

# Parser for command-line arguments.
parser = argparse.ArgumentParser(
    description=__doc__,
    formatter_class=argparse.RawDescriptionHelpFormatter,
    parents=[tools.argparser])
parser.add_argument('-su', '--servlet_url', help='the URL of the servlet that parses the JSON files', required=True)
parser.add_argument('-if', '--input_file', help='an input JSON file', required=True)

# CLIENT_SECRETS is name of a file containing the OAuth 2.0 information for this
# application, including client_id and client_secret. You can see the Client ID
# and Client secret on the APIs page in the Cloud Console:
# <https://cloud.google.com/console#/project/1083819269171/apiui>
CLIENT_SECRETS = os.path.join(os.path.dirname(__file__), 'client_secrets.json')

# Set up a Flow object to be used for authentication.
# Add one or more of the following scopes. PLEASE ONLY ADD THE SCOPES YOU
# NEED. For more information on using scopes please see
# <https://developers.google.com/+/best-practices>.
FLOW = client.flow_from_clientsecrets(CLIENT_SECRETS,
  scope=[
      'https://www.googleapis.com/auth/freebase',
    ],
    message=tools.message_if_missing(CLIENT_SECRETS))


def main(argv):
  # Parse the command-line flags.
  flags = parser.parse_args(argv[1:])
  #define some vars 
  service_url_write = 'https://www.googleapis.com/freebase/v1sandbox/mqlwrite'
  inputFile = flags.input_file
  servlet_url = flags.servlet_url

  # If the credentials don't exist or are invalid run through the native client
  # flow. The Storage object will ensure that if successful the good
  # credentials will get written back to the file.
  storage = file.Storage('sample.dat')
  credentials = storage.get()
  if credentials is None or credentials.invalid:
    credentials = tools.run_flow(FLOW, storage, flags)

  # Create an httplib2.Http object to handle our HTTP requests and authorize it
  # with our good Credentials.
  http = httplib2.Http()
  http = credentials.authorize(http)
  # Construct the service object for the interacting with the Freebase API.
  service = discovery.build('freebase', 'v1', http=http)

  if inputFile:
    ifp =  os.path.dirname(inputFile)
    ifn = os.path.basename(inputFile)
    start = time.time()
    cleanJson = getCleanJson(servlet_url, ifp ,ifn)
    if cleanJson:
      #now prepare a write query for the cleanJSON
      se_mid = writeToFreebase(cleanJson, service_url_write, http, credentials)
      print "created selex experiment topic with mid: "+se_mid["mid"]
      end = time.time()
      tt = end-start
      print "time elapsed in seconds: "+str(tt)

def writeToFreebase(cleanJson, aServiceUrl, anHttp, someCredentials):
  #create an empty selex experiment topic and get its mid
  mid_dict = createSelexExperimentTopic(aServiceUrl, anHttp, someCredentials)
  #add the reference details from this experiment
  addReferenceDetails(mid_dict, cleanJson, aServiceUrl, anHttp, someCredentials)
  addSelexDetails(mid_dict, cleanJson, aServiceUrl, anHttp, someCredentials)
  addSelexConditions(mid_dict, cleanJson, aServiceUrl, anHttp,someCredentials)
  addInteractions(mid_dict["mid"], cleanJson, aServiceUrl, anHttp, someCredentials)
  return mid_dict

#Creates a empty interaction topic including and returns the mid
# and connects it to the given selex experiment mid
def createInteractionTopic(aSelexExperimentMid, aServiceUrl, anHttp, someCredentials):
  q = {
    "create":"unconditional",
    "mid":None,
    "type":"/base/aptamer/interaction",
    "b:type":"/base/aptamer/experimental_outcome",
    "/base/aptamer/experimental_outcome/is_outcome_of":{
      "connect":"insert",
      "mid" : aSelexExperimentMid
    }
  }
  params = makeRequestBody(someCredentials, q)
  r = runQuery(params, aServiceUrl, anHttp)
  if r == None:
    raise Exception ("Could not create interaction topic!")
    sys.exit()
  return r

{'fraction_non_alphanumeric': 0.08138111174597992, 'fraction_numerical': 0.006040323238919272, 'mean_word_length': 4.494170403587444, 'pattern_counts': {'":': 12, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 8, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2924291', 'n_tokens_mistral': 3566, 'n_tokens_neox': 3450, 'n_words': 1362}