Storage Examples
Specify meta-data when uploading an object
Most of the providers allow you to associate arbitrary key-value pairs (meta-data) with every uploaded object. This example shows how to do that in Libcloud.
from libcloud.storage.types import Provider
from libcloud.storage.providers import get_driver
FILE_PATH = "/home/user/myfile.tar.gz"
cls = get_driver(Provider.S3)
driver = cls("api key", "api secret key")
container = driver.get_container(container_name="my-backups-12345")
extra = {"meta_data": {"owner": "myuser", "created": "2014-02-2"}}
with open(FILE_PATH, "rb") as iterator:
obj = driver.upload_object_via_stream(
iterator=iterator, container=container, object_name="backup.tar.gz", extra=extra
)
As you can see in the example above, meta-data is specified by including
meta_data
key in the extra
dictionary argument which gets passed
to all the upload methods.
Download part of an object
To perform a partial (range) object download, you can utilize new
libcloud.storage.base.StorageDriver.download_object_range()
and libcloud.storage.base.StorageDriver.download_object_range_as_stream()
methods which have been added in Libcloud v3.0.0.
start_bytes
and end_bytes
behave in the same manner as Python
indexing which means that start_bytes
is inclusive and end_bytes
is non-inclusive.
For example, if the object content is 0123456789
, here is what
would be returned for various values of start and end bytes arguments:
start_bytes=0, end_bytes=1 -> 0
start_bytes=0, end_bytes=2 -> 01
start_bytes=5, end_bytes=6 -> 5
start_bytes=5, end_bytes=10 -> 56789
from libcloud.storage.types import Provider
from libcloud.storage.providers import get_driver
Driver = get_driver(Provider.AZURE_BLOBS)
driver = Driver("storagename", "key")
container = driver.list_containers()[0]
obj = container.list_objects()[0]
# Download first 5 bytes of an object (aka bytes 0, 1, 2, 3 and 4)
print(next(driver.download_object_range_as_stream(obj=obj, start_bytes=0, end_bytes=5)))
# Download bytes 5-8 (inclusive) of an object
print(next(driver.download_object_range_as_stream(obj=obj, start_bytes=5, end_bytes=9)))
Create a backup of a directory and directly stream it to CloudFiles
import subprocess
from datetime import datetime
from libcloud.storage.types import Provider, ContainerDoesNotExistError
from libcloud.storage.providers import get_driver
driver = get_driver(Provider.CLOUDFILES_US)("username", "api key")
directory = "/home/some/path"
cmd = "tar cvzpf - %s" % (directory)
object_name = "backup-%s.tar.gz" % (datetime.now().strftime("%Y-%m-%d"))
container_name = "backups"
# Create a container if it doesn't already exist
try:
container = driver.get_container(container_name=container_name)
except ContainerDoesNotExistError:
container = driver.create_container(container_name=container_name)
pipe = subprocess.Popen(cmd, bufsize=0, shell=True, stdout=subprocess.PIPE)
return_code = pipe.poll()
print("Uploading object...")
while return_code is None:
# Compress data in our directory and stream it directly to CF
obj = container.upload_object_via_stream(iterator=pipe.stdout, object_name=object_name)
return_code = pipe.poll()
print("Upload complete, transferred: %s KB" % (obj.size / 1024))
Efficiently download multiple files using gevent
import os.path
from gevent import monkey # pylint: disable=import-error
from gevent.pool import Pool # pylint: disable=import-error
from libcloud.storage.types import Provider
from libcloud.storage.providers import get_driver
monkey.patch_all()
USERNAME = "username"
API_KEY = "api key"
cls = get_driver(Provider.CLOUDFILES_US)
driver = cls(USERNAME, API_KEY)
def download_obj(container, obj):
driver = cls(USERNAME, API_KEY)
obj = driver.get_object(container_name=container.name, object_name=obj.name)
filename = os.path.basename(obj.name)
path = os.path.join(os.path.expanduser("~/Downloads"), filename)
print("Downloading: {} to {}".format(obj.name, path))
obj.download(destination_path=path)
containers = driver.list_containers()
jobs = []
pool = Pool(20)
for index, container in enumerate(containers):
objects = container.list_objects()
for obj in objects:
pool.spawn(download_obj, container, obj)
pool.join()
print("Done")
Publishing a static website using CloudFiles driver
Note
This example works with Libcloud version 0.11.0 and above.
from StringIO import StringIO
from libcloud.storage.types import Provider
from libcloud.storage.providers import get_driver
CloudFiles = get_driver(Provider.CLOUDFILES_US)
driver = CloudFiles("username", "api key")
container = driver.create_container(container_name="my_website")
iterator1 = StringIO("<p>Hello World from Libcloud!</p>")
iterator2 = StringIO("<p>Oh, noez, 404!!</p>")
iterator3 = StringIO("<p>Oh, noez, 401!!</p>")
driver.upload_object_via_stream(iterator=iterator1, container=container, object_name="index.html")
driver.upload_object_via_stream(
iterator=iterator2, container=container, object_name="404error.html"
)
driver.upload_object_via_stream(
iterator=iterator3, container=container, object_name="401error.html"
)
driver.ex_enable_static_website(container=container)
driver.ex_set_error_page(container=container, file_name="error.html")
driver.enable_container_cdn(container=container)
print(
"All done you can view the website at: %s" % (driver.get_container_cdn_url(container=container))
)