Storage Examples

Specify meta-data when uploading an object

Most of the providers allow you to associate arbitrary key-value pairs (meta-data) with every uploaded object. This example shows how to do that in Libcloud.

from libcloud.storage.types import Provider
from libcloud.storage.providers import get_driver

FILE_PATH = '/home/user/myfile.tar.gz'

cls = get_driver(Provider.S3)
driver = cls('api key', 'api secret key')

container = driver.get_container(container_name='my-backups-12345')

extra = {'meta_data': {'owner': 'myuser', 'created': '2014-02-2'}}

with open(FILE_PATH, 'rb') as iterator:
    obj = driver.upload_object_via_stream(iterator=iterator,
                                          container=container,
                                          object_name='backup.tar.gz',
                                          extra=extra)

As you can see in the example above, meta-data is specified by including meta_data key in the extra dictionary argument which gets passed to all the upload methods.

Download part of an object

To perform a partial (range) object download, you can utilize new libcloud.storage.base.StorageDriver.download_object_range() and libcloud.storage.base.StorageDriver.download_object_range_as_stream() methods which have been added in Libcloud v3.0.0.

start_bytes and end_bytes behave in the same manner as Python indexing which means that start_bytes is inclusive and end_bytes is non-inclusive.

For example, if the object content is 0123456789, here is what would be returned for various values of start and end bytes arguments:

  • start_bytes=0, end_bytes=1 -> 0
  • start_bytes=0, end_bytes=2 -> 01
  • start_bytes=5, end_bytes=6 -> 5
  • start_bytes=5, end_bytes=10 -> 56789
from libcloud.storage.types import Provider
from libcloud.storage.providers import get_driver

Driver = get_driver(Provider.AZURE_BLOBS)
driver = Driver('storagename', 'key')

container = driver.list_containers()[0]
obj = container.list_objects()[0]

# Download first 5 bytes of an object (aka bytes 0, 1, 2, 3 and 4)
print(next(driver.download_object_range_as_stream(obj=obj, start_bytes=0,
                                                  end_bytes=5)))

# Download bytes 5-8 (inclusive) of an object
print(next(driver.download_object_range_as_stream(obj=obj, start_bytes=5,
                                                  end_bytes=9)))

Create a backup of a directory and directly stream it to CloudFiles

import subprocess
from datetime import datetime

from libcloud.storage.types import Provider, ContainerDoesNotExistError
from libcloud.storage.providers import get_driver

driver = get_driver(Provider.CLOUDFILES_US)('username', 'api key')

directory = '/home/some/path'
cmd = 'tar cvzpf - %s' % (directory)

object_name = 'backup-%s.tar.gz' % (datetime.now().strftime('%Y-%m-%d'))
container_name = 'backups'

# Create a container if it doesn't already exist
try:
    container = driver.get_container(container_name=container_name)
except ContainerDoesNotExistError:
    container = driver.create_container(container_name=container_name)

pipe = subprocess.Popen(cmd, bufsize=0, shell=True, stdout=subprocess.PIPE)
return_code = pipe.poll()

print('Uploading object...')

while return_code is None:
    # Compress data in our directory and stream it directly to CF
    obj = container.upload_object_via_stream(iterator=pipe.stdout,
                                             object_name=object_name)
    return_code = pipe.poll()

print('Upload complete, transferred: %s KB' % ((obj.size / 1024)))

Efficiently download multiple files using gevent

import os.path

from gevent import monkey  # pylint: disable=import-error
from gevent.pool import Pool  # pylint: disable=import-error
monkey.patch_all()

from libcloud.storage.providers import get_driver
from libcloud.storage.types import Provider

USERNAME = 'username'
API_KEY = 'api key'

cls = get_driver(Provider.CLOUDFILES_US)
driver = cls(USERNAME, API_KEY)


def download_obj(container, obj):
    driver = cls(USERNAME, API_KEY)
    obj = driver.get_object(container_name=container.name,
                            object_name=obj.name)
    filename = os.path.basename(obj.name)
    path = os.path.join(os.path.expanduser('~/Downloads'), filename)
    print('Downloading: %s to %s' % (obj.name, path))
    obj.download(destination_path=path)


containers = driver.list_containers()

jobs = []
pool = Pool(20)

for index, container in enumerate(containers):
    objects = container.list_objects()

    for obj in objects:
        pool.spawn(download_obj, container, obj)

pool.join()
print('Done')

Publishing a static website using CloudFiles driver

Note

This example works with Libcloud version 0.11.0 and above.

from StringIO import StringIO

from libcloud.storage.types import Provider
from libcloud.storage.providers import get_driver

CloudFiles = get_driver(Provider.CLOUDFILES_US)

driver = CloudFiles('username', 'api key')

container = driver.create_container(container_name='my_website')

iterator1 = StringIO('<p>Hello World from Libcloud!</p>')
iterator2 = StringIO('<p>Oh, noez, 404!!</p>')
iterator3 = StringIO('<p>Oh, noez, 401!!</p>')

driver.upload_object_via_stream(iterator=iterator1, container=container,
                                object_name='index.html')
driver.upload_object_via_stream(iterator=iterator2, container=container,
                                object_name='404error.html')
driver.upload_object_via_stream(iterator=iterator3, container=container,
                                object_name='401error.html')

driver.ex_enable_static_website(container=container)
driver.ex_set_error_page(container=container, file_name='error.html')
driver.enable_container_cdn(container=container)

print('All done you can view the website at: %s' % (
      driver.get_container_cdn_url(container=container)))