<jupyter_start><jupyter_text>Storage API sample<jupyter_code>import gcp
import gcp.storage as storage
from gcp.context import Context
import random
import pandas as pd
from StringIO import StringIO
project = Context.default().project_id
bucket_name = "yukoga-kaggle"
bucket_path = "gs://" + bucket_name
test_sample_size = 1000
train_sample_size = 1000
sample_submission_sample_size = 1000
%%bash 
curl --silent -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/service-accounts/default/email
# get skiprows for pandas.DataFrame
def get_skiprows(sample_size, num_records):
  return sorted(random.sample(range(1,num_records),num_records - sample_size))
# test data
%storage read --object "gs://yukoga-kaggle/facebook-checkin/test.csv" --variable tmp_table
num_records = len(tmp_table.split('\n'))
test = pd.read_csv(StringIO(tmp_table), skiprows=get_skiprows(test_sample_size, num_records))
del tmp_table
test.head()
# get sampled records from cloud storage
def read_sampled_lines(item, sample_size):
  """Reads the content of this item as text, and return a list of lines up to some max.

  Args:
    item: item object from Google Cloud Storage.
    start_offset_line: an index indicates start offset records within a item.
    max_lines: max number of lines to return. If None, return all lines.
  Returns:
    The text content of the item as a list of lines.
  Raises:
    Exception if there was an error requesting the item's content.
  """
  def read_specific_lines(item, offset, num_records):
    start_to_read = 100 * (0 if offset is None else offset)
    max_to_read = item.metadata.size
    num_records = max_to_read if num_records is None else num_records
    bytes_to_read = min(100 * num_records, item.metadata.size)
    
    lines = []
    while True:
      content = item.read_from(start_offset=start_to_read, bytes_to_read)
      lines = content.split('\n')
      if len(lines) > num_records or bytes_to_read >= max_to_read:
        break
      bytes_to_read = min_lines or bytes_to_read >= max_to_read:

    del lines[-1]
    return lines[0:num_records]

mybucket = storage.Bucket(bucket_name)
for item in mybucket.items():
  print item.metadata.name + " : " + str(item.metadata.size)
  
help(item.read_lines)
import inspect
print inspect.getsource(item._api.object_download)
print inspect.getsource(item.read_from)
print inspect.getsource(item.read_lines)
print inspect.getsource(gcp._util.Http.request)
# test data
%storage read --object "gs://yukoga-kaggle/facebook-checkin/test.csv" --variable tmp_table
num_records = len(tmp_table.split('\n'))
test = pd.read_csv(StringIO(tmp_table), skiprows=get_skiprows(test_sample_size, num_records))
del tmp_table

# sample submission data
%storage read --object "gs://yukoga-kaggle/facebook-checkin/sample_submission.csv" --variable tmp_table
num_records = len(tmp_table.split('\n'))
sample_submission = pd.read_csv(StringIO(tmp_table), skiprows=get_skiprows(sample_submission_sample_size, num_records))
del tmp_table

# train data
%storage read --object "gs://yukoga-kaggle/facebook-checkin/train.csv" --variable tmp_table
num_records = len(tmp_table.split('\n'))
train = pd.read_csv(StringIO(tmp_table), skiprows=get_skiprows(train_sample_size, num_records))
del tmp_table<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10167427701674277, 'fraction_numerical': 0.007001522070015221, 'mean_word_length': 5.096474953617811, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '554353', 'license': 'Apache-2.0', 'n_tokens_mistral': 1088, 'n_tokens_neox': 1066, 'n_words': 269, 'path': 'notebooks/Storage API sample.ipynb', 'repo_name': 'yukoga/datalab-archives', 'size': '3285'}