Usage

To use gcs-client in a project you will need to have Credentials to access intended Google Cloud Storage.

Credentials are generated in Google Developers Console in the Credentials section of the API Manager of the project. Recommended credentials file is JSON.

Once you have the credentials you can start using gcs_client to access your project.

Listing buckets

import gcs_client

credentials_file = 'private_key.json'
project_name = 'project_name'

credentials = gcs_client.Credentials(credentials_file)
project = gcs_client.Project(project_name, credentials)

buckets = project.list()
print 'Buckets:\n\t- ','\n\t- '.join(map(str, buckets))

Creating a bucket

import gcs_client

credentials_file = 'private_key.json'
project_name = 'project_name'

credentials = gcs_client.Credentials(credentials_file)
project = gcs_client.Project(project_name, credentials)

bucket = project.create_bucket('my_new_bucket', location='EU')
print 'Bucket %s is located in %s with storage class %s' % (bucket, bucket.location,
                                                            bucket.storageClass)

Deleting a bucket

import gcs_client

credentials_file = 'private_key.json'
project_name = 'project_name'

credentials = gcs_client.Credentials(credentials_file)
project = gcs_client.Project(project_name, credentials)

buckets = project.list()
# Delete one bucket but never the default bucket
default_bucket = project.default_bucket_name
filtered = filter(lambda b: b.name != default_bucket, buckets)
if filtered:
    buckets[0].delete()

Listing all objects

import gcs_client

credentials_file = 'private_key.json'
project_name = 'project_name'

credentials = gcs_client.Credentials(credentials_file)
project = gcs_client.Project(project_name, credentials)

buckets = project.list()
objects = buckets[0].list()

print 'Contents of bucket %s:' % bucket
if objects:
    print '\t','\n\t'.join(map(lambda o: o.name + ' has %s bytes' % o.size, objects))
else:
    print '\tThere are no objects'

Listing objects with a prefix

import gcs_client

credentials = gcs_client.Credentials('private_key.json')
bucket = gcs_client.Bucket('bucket_name', credentials)
directory = 'var/log'
objects = bucket.list(directory)

print 'Contents of %s/%s:' % (bucket.name, directory)
if objects:
    print '\t','\n\t'.join(map(lambda o: o.name + ' has %s bytes' % o.size, objects))
else:
    print '\tThere are no objects'

List contents of a bucket as a directory

We can list a bucket as if it were a directory by passing delimiter optional argument on the list call.

Following example implements tree command for a bucket.

This is just for demonstration purposes, since it is not efficient because for each “directory” we find we make another request to the server to list its contents. It would be more efficient to request all the objects in one go and then rebuild the tree locally.

import gcs_client

def print_obj(obj, i, last):
    if isinstance(obj, gcs_client.Prefix):
        name = obj.prefix.split('/')[-2]
    else:
        name = obj.name.split('/')[-1]
    print ('   ' * i) + ('└──' if last else '├──'), name

def tree(objs, indent=0):
    if indent == 0:
        print '.'
    for i in range(len(objs)):
        obj = objs[i]
        print_obj(obj, indent, i == len(objs) - 1)
        if isinstance(obj, gcs_client.Prefix):
            tree(obj.list(), indent+1)

credentials = gcs_client.Credentials('private_key.json')
bucket = gcs_client.Bucket('bucket_name', credentials)

print '$ tree', bucket.name
tree(bucket.list(delimiter='/'))

Deleting objects

import gcs_client

credentials_file = 'private_key.json'
project_name = 'project_name'

credentials = gcs_client.Credentials(credentials_file)
project = gcs_client.Project(project_name, credentials)

bucket = project.list()[0]
objects = bucket.list()

if objects:
    obj = objects[0]
    print 'Deleting object %s' % obj
    obj.delete()

Reading objects

import gcs_client

credentials_file = 'private_key.json'
project_name = 'project_name'

credentials = gcs_client.Credentials(credentials_file)
project = gcs_client.Project(project_name, credentials)

buckets = project.list()
objects = buckets[0].list()

if objects:
    with objects[0].open() as obj:
        print 'Contents of file %s are:\n' % obj.name, obj.read()

Reading objects in big chunks

import gcs_client

credentials = gcs_client.Credentials('private_key.json')
bucket = gcs_client.Bucket('bucket_name', credentials)

chunksize = 4 * 1024 * 1024

with bucket.open('my_file', 'r', chunksize=chunksize) as obj:
    print 'Contents of file %s are:\n' % obj.name, obj.read()

Writing objects

import gcs_client

credentials_file = 'private_key.json'
project_name = 'project_name'

credentials = gcs_client.Credentials(credentials_file)
project = gcs_client.Project(project_name, credentials)

bucket = project.list()[0]

with bucket.open('new_file.txt', 'w') as obj:
    obj.write('Hello world\n')

with bucket.open('new_file.txt') as obj:
    print obj.read()

Changing default retry configuration

All operations use retries with Truncated Exponential Backoff by default, but we can change default configuration.

import gcs_client

# Set default retry configuration using a RetryParams instance
new_retry_cfg = gcs_client.RetryParams(max_retries=10, initial_delay=0.5, max_backoff=8,
                                       randomize=False)
gcs_client.RetryParams.set_default(new_retry_cfg)

# Set default retry configuration via params
gcs_client.RetryParams.set_default(max_retries=10, initial_delay=0.5, max_backoff=8,
                                   randomize=False)

Disabling default retries

We may want to disable all retries for all instances that are using default retry configuration. Those that are using specific instance configurations will continue doing so.

import gcs_client

# Disable retry configuration
gcs_client.RetryParams.set_default(0)

Per instance retry configuration

We can set specific retry configuration for an instance. Important to notice that listed objects will inherit retry configuration from the object that did the listing.

import gcs_client

credentials_file = 'private_key.json'
project_name = 'project_name'

credentials = gcs_client.Credentials(credentials_file)
project = gcs_client.Project(project_name, credentials)

bucket = project.list()[0]
# Set bucket retry configuration
bucket.retry_params = gcs_client.RetryParams(max_retries=10, initial_delay=0.5, max_backoff=8,
                                             randomize=False)

# Disable retries on the bucket
bucket.retry_params = None