Storage Examples¶
Specify meta-data when uploading an object¶
Most of the providers allow you to associate arbitrary key-value pairs (meta-data) with every uploaded object. This example shows how to do that in Libcloud.
from libcloud.storage.types import Provider
from libcloud.storage.providers import get_driver
FILE_PATH = '/home/user/myfile.tar.gz'
cls = get_driver(Provider.S3)
driver = cls('api key', 'api secret key')
container = driver.get_container(container_name='my-backups-12345')
extra = {'meta_data': {'owner': 'myuser', 'created': '2014-02-2'}}
with open(FILE_PATH, 'rb') as iterator:
obj = driver.upload_object_via_stream(iterator=iterator,
container=container,
object_name='backup.tar.gz',
extra=extra)
As you can see in the example above, meta-data is specified by including
meta_data
key in the extra
dictionary argument which gets passed
to all the upload methods.
Create a backup of a directory and directly stream it to CloudFiles¶
import subprocess
from datetime import datetime
from libcloud.storage.types import Provider, ContainerDoesNotExistError
from libcloud.storage.providers import get_driver
driver = get_driver(Provider.CLOUDFILES_US)('username', 'api key')
directory = '/home/some/path'
cmd = 'tar cvzpf - %s' % (directory)
object_name = 'backup-%s.tar.gz' % (datetime.now().strftime('%Y-%m-%d'))
container_name = 'backups'
# Create a container if it doesn't already exist
try:
container = driver.get_container(container_name=container_name)
except ContainerDoesNotExistError:
container = driver.create_container(container_name=container_name)
pipe = subprocess.Popen(cmd, bufsize=0, shell=True, stdout=subprocess.PIPE)
return_code = pipe.poll()
print 'Uploading object...'
while return_code is None:
# Compress data in our directory and stream it directly to CF
obj = container.upload_object_via_stream(iterator=pipe.stdout,
object_name=object_name)
return_code = pipe.poll()
print 'Upload complete, transferred: %s KB' % ((obj.size / 1024))
Efficiently download multiple files using gevent¶
import os.path
from gevent import monkey
from gevent.pool import Pool
monkey.patch_all()
from libcloud.storage.providers import get_driver
from libcloud.storage.types import Provider
USERNAME = 'username'
API_KEY = 'api key'
cls = get_driver(Provider.CLOUDFILES_US)
driver = cls(USERNAME, API_KEY)
def download_obj(container, obj):
driver = cls(USERNAME, API_KEY)
obj = driver.get_object(container_name=container.name,
object_name=obj.name)
filename = os.path.basename(obj.name)
path = os.path.join(os.path.expanduser('~/Downloads'), filename)
print 'Downloading: %s to %s' % (obj.name, path)
obj.download(destination_path=path)
containers = driver.list_containers()
jobs = []
pool = Pool(20)
for index, container in enumerate(containers):
objects = container.list_objects()
for obj in objects:
pool.spawn(download_obj, container, obj)
pool.join()
print 'Done'
Publishing a static website using CloudFiles driver¶
Note
This example works with Libcloud version 0.11.0 and above.
from StringIO import StringIO
from libcloud.storage.types import Provider
from libcloud.storage.providers import get_driver
CloudFiles = get_driver(Provider.CLOUDFILES_US)
driver = CloudFiles('username', 'api key')
container = driver.create_container(container_name='my_website')
iterator1 = StringIO('<p>Hello World from Libcloud!</p>')
iterator2 = StringIO('<p>Oh, noez, 404!!</p>')
iterator3 = StringIO('<p>Oh, noez, 401!!</p>')
driver.upload_object_via_stream(iterator=iterator1, container=container,
object_name='index.html')
driver.upload_object_via_stream(iterator=iterator2, container=container,
object_name='404error.html')
driver.upload_object_via_stream(iterator=iterator3, container=container,
object_name='401error.html')
driver.ex_enable_static_website(container=container)
driver.ex_set_error_page(container=container, file_name='error.html')
driver.enable_container_cdn(container=container)
print('All done you can view the website at: ' +
driver.get_container_cdn_url(container=container))