Revonzy Mini Shell
# Copyright (c) 2012 Mitch Garnaat http://garnaat.org/
# Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish, dis-
# tribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the fol-
# lowing conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#
from boto.dynamodb.batch import BatchList
from boto.dynamodb.schema import Schema
from boto.dynamodb.item import Item
from boto.dynamodb import exceptions as dynamodb_exceptions
import time
class TableBatchGenerator(object):
"""
A low-level generator used to page through results from
batch_get_item operations.
:ivar consumed_units: An integer that holds the number of
ConsumedCapacityUnits accumulated thus far for this
generator.
"""
def __init__(self, table, keys, attributes_to_get=None,
consistent_read=False):
self.table = table
self.keys = keys
self.consumed_units = 0
self.attributes_to_get = attributes_to_get
self.consistent_read = consistent_read
def _queue_unprocessed(self, res):
if u'UnprocessedKeys' not in res:
return
if self.table.name not in res[u'UnprocessedKeys']:
return
keys = res[u'UnprocessedKeys'][self.table.name][u'Keys']
for key in keys:
h = key[u'HashKeyElement']
r = key[u'RangeKeyElement'] if u'RangeKeyElement' in key else None
self.keys.append((h, r))
def __iter__(self):
while self.keys:
# Build the next batch
batch = BatchList(self.table.layer2)
batch.add_batch(self.table, self.keys[:100],
self.attributes_to_get)
res = batch.submit()
# parse the results
if self.table.name not in res[u'Responses']:
continue
self.consumed_units += res[u'Responses'][self.table.name][u'ConsumedCapacityUnits']
for elem in res[u'Responses'][self.table.name][u'Items']:
yield elem
# re-queue un processed keys
self.keys = self.keys[100:]
self._queue_unprocessed(res)
class Table(object):
"""
An Amazon DynamoDB table.
:ivar name: The name of the table.
:ivar create_time: The date and time that the table was created.
:ivar status: The current status of the table. One of:
'ACTIVE', 'UPDATING', 'DELETING'.
:ivar schema: A :class:`boto.dynamodb.schema.Schema` object representing
the schema defined for the table.
:ivar item_count: The number of items in the table. This value is
set only when the Table object is created or refreshed and
may not reflect the actual count.
:ivar size_bytes: Total size of the specified table, in bytes.
Amazon DynamoDB updates this value approximately every six hours.
Recent changes might not be reflected in this value.
:ivar read_units: The ReadCapacityUnits of the tables
Provisioned Throughput.
:ivar write_units: The WriteCapacityUnits of the tables
Provisioned Throughput.
:ivar schema: The Schema object associated with the table.
"""
def __init__(self, layer2, response):
"""
:type layer2: :class:`boto.dynamodb.layer2.Layer2`
:param layer2: A `Layer2` api object.
:type response: dict
:param response: The output of
`boto.dynamodb.layer1.Layer1.describe_table`.
"""
self.layer2 = layer2
self._dict = {}
self.update_from_response(response)
@classmethod
def create_from_schema(cls, layer2, name, schema):
"""Create a Table object.
If you know the name and schema of your table, you can
create a ``Table`` object without having to make any
API calls (normally an API call is made to retrieve
the schema of a table).
Example usage::
table = Table.create_from_schema(
boto.connect_dynamodb(),
'tablename',
Schema.create(hash_key=('keyname', 'N')))
:type layer2: :class:`boto.dynamodb.layer2.Layer2`
:param layer2: A ``Layer2`` api object.
:type name: str
:param name: The name of the table.
:type schema: :class:`boto.dynamodb.schema.Schema`
:param schema: The schema associated with the table.
:rtype: :class:`boto.dynamodb.table.Table`
:return: A Table object representing the table.
"""
table = cls(layer2, {'Table': {'TableName': name}})
table._schema = schema
return table
def __repr__(self):
return 'Table(%s)' % self.name
@property
def name(self):
return self._dict['TableName']
@property
def create_time(self):
return self._dict.get('CreationDateTime', None)
@property
def status(self):
return self._dict.get('TableStatus', None)
@property
def item_count(self):
return self._dict.get('ItemCount', 0)
@property
def size_bytes(self):
return self._dict.get('TableSizeBytes', 0)
@property
def schema(self):
return self._schema
@property
def read_units(self):
try:
return self._dict['ProvisionedThroughput']['ReadCapacityUnits']
except KeyError:
return None
@property
def write_units(self):
try:
return self._dict['ProvisionedThroughput']['WriteCapacityUnits']
except KeyError:
return None
def update_from_response(self, response):
"""
Update the state of the Table object based on the response
data received from Amazon DynamoDB.
"""
# 'Table' is from a describe_table call.
if 'Table' in response:
self._dict.update(response['Table'])
# 'TableDescription' is from a create_table call.
elif 'TableDescription' in response:
self._dict.update(response['TableDescription'])
if 'KeySchema' in self._dict:
self._schema = Schema(self._dict['KeySchema'])
def refresh(self, wait_for_active=False, retry_seconds=5):
"""
Refresh all of the fields of the Table object by calling
the underlying DescribeTable request.
:type wait_for_active: bool
:param wait_for_active: If True, this command will not return
until the table status, as returned from Amazon DynamoDB, is
'ACTIVE'.
:type retry_seconds: int
:param retry_seconds: If wait_for_active is True, this
parameter controls the number of seconds of delay between
calls to update_table in Amazon DynamoDB. Default is 5 seconds.
"""
done = False
while not done:
response = self.layer2.describe_table(self.name)
self.update_from_response(response)
if wait_for_active:
if self.status == 'ACTIVE':
done = True
else:
time.sleep(retry_seconds)
else:
done = True
def update_throughput(self, read_units, write_units):
"""
Update the ProvisionedThroughput for the Amazon DynamoDB Table.
:type read_units: int
:param read_units: The new value for ReadCapacityUnits.
:type write_units: int
:param write_units: The new value for WriteCapacityUnits.
"""
self.layer2.update_throughput(self, read_units, write_units)
def delete(self):
"""
Delete this table and all items in it. After calling this
the Table objects status attribute will be set to 'DELETING'.
"""
self.layer2.delete_table(self)
def get_item(self, hash_key, range_key=None,
attributes_to_get=None, consistent_read=False,
item_class=Item):
"""
Retrieve an existing item from the table.
:type hash_key: int|long|float|str|unicode|Binary
:param hash_key: The HashKey of the requested item. The
type of the value must match the type defined in the
schema for the table.
:type range_key: int|long|float|str|unicode|Binary
:param range_key: The optional RangeKey of the requested item.
The type of the value must match the type defined in the
schema for the table.
:type attributes_to_get: list
:param attributes_to_get: A list of attribute names.
If supplied, only the specified attribute names will
be returned. Otherwise, all attributes will be returned.
:type consistent_read: bool
:param consistent_read: If True, a consistent read
request is issued. Otherwise, an eventually consistent
request is issued.
:type item_class: Class
:param item_class: Allows you to override the class used
to generate the items. This should be a subclass of
:class:`boto.dynamodb.item.Item`
"""
return self.layer2.get_item(self, hash_key, range_key,
attributes_to_get, consistent_read,
item_class)
lookup = get_item
def has_item(self, hash_key, range_key=None, consistent_read=False):
"""
Checks the table to see if the Item with the specified ``hash_key``
exists. This may save a tiny bit of time/bandwidth over a
straight :py:meth:`get_item` if you have no intention to touch
the data that is returned, since this method specifically tells
Amazon not to return anything but the Item's key.
:type hash_key: int|long|float|str|unicode|Binary
:param hash_key: The HashKey of the requested item. The
type of the value must match the type defined in the
schema for the table.
:type range_key: int|long|float|str|unicode|Binary
:param range_key: The optional RangeKey of the requested item.
The type of the value must match the type defined in the
schema for the table.
:type consistent_read: bool
:param consistent_read: If True, a consistent read
request is issued. Otherwise, an eventually consistent
request is issued.
:rtype: bool
:returns: ``True`` if the Item exists, ``False`` if not.
"""
try:
# Attempt to get the key. If it can't be found, it'll raise
# an exception.
self.get_item(hash_key, range_key=range_key,
# This minimizes the size of the response body.
attributes_to_get=[hash_key],
consistent_read=consistent_read)
except dynamodb_exceptions.DynamoDBKeyNotFoundError:
# Key doesn't exist.
return False
return True
def new_item(self, hash_key=None, range_key=None, attrs=None,
item_class=Item):
"""
Return an new, unsaved Item which can later be PUT to
Amazon DynamoDB.
This method has explicit (but optional) parameters for
the hash_key and range_key values of the item. You can use
these explicit parameters when calling the method, such as::
>>> my_item = my_table.new_item(hash_key='a', range_key=1,
attrs={'key1': 'val1', 'key2': 'val2'})
>>> my_item
{u'bar': 1, u'foo': 'a', 'key1': 'val1', 'key2': 'val2'}
Or, if you prefer, you can simply put the hash_key and range_key
in the attrs dictionary itself, like this::
>>> attrs = {'foo': 'a', 'bar': 1, 'key1': 'val1', 'key2': 'val2'}
>>> my_item = my_table.new_item(attrs=attrs)
>>> my_item
{u'bar': 1, u'foo': 'a', 'key1': 'val1', 'key2': 'val2'}
The effect is the same.
.. note:
The explicit parameters take priority over the values in
the attrs dict. So, if you have a hash_key or range_key
in the attrs dict and you also supply either or both using
the explicit parameters, the values in the attrs will be
ignored.
:type hash_key: int|long|float|str|unicode|Binary
:param hash_key: The HashKey of the new item. The
type of the value must match the type defined in the
schema for the table.
:type range_key: int|long|float|str|unicode|Binary
:param range_key: The optional RangeKey of the new item.
The type of the value must match the type defined in the
schema for the table.
:type attrs: dict
:param attrs: A dictionary of key value pairs used to
populate the new item.
:type item_class: Class
:param item_class: Allows you to override the class used
to generate the items. This should be a subclass of
:class:`boto.dynamodb.item.Item`
"""
return item_class(self, hash_key, range_key, attrs)
def query(self, hash_key, *args, **kw):
"""
Perform a query on the table.
:type hash_key: int|long|float|str|unicode|Binary
:param hash_key: The HashKey of the requested item. The
type of the value must match the type defined in the
schema for the table.
:type range_key_condition: :class:`boto.dynamodb.condition.Condition`
:param range_key_condition: A Condition object.
Condition object can be one of the following types:
EQ|LE|LT|GE|GT|BEGINS_WITH|BETWEEN
The only condition which expects or will accept two
values is 'BETWEEN', otherwise a single value should
be passed to the Condition constructor.
:type attributes_to_get: list
:param attributes_to_get: A list of attribute names.
If supplied, only the specified attribute names will
be returned. Otherwise, all attributes will be returned.
:type request_limit: int
:param request_limit: The maximum number of items to retrieve
from Amazon DynamoDB on each request. You may want to set
a specific request_limit based on the provisioned throughput
of your table. The default behavior is to retrieve as many
results as possible per request.
:type max_results: int
:param max_results: The maximum number of results that will
be retrieved from Amazon DynamoDB in total. For example,
if you only wanted to see the first 100 results from the
query, regardless of how many were actually available, you
could set max_results to 100 and the generator returned
from the query method will only yeild 100 results max.
:type consistent_read: bool
:param consistent_read: If True, a consistent read
request is issued. Otherwise, an eventually consistent
request is issued.
:type scan_index_forward: bool
:param scan_index_forward: Specified forward or backward
traversal of the index. Default is forward (True).
:type exclusive_start_key: list or tuple
:param exclusive_start_key: Primary key of the item from
which to continue an earlier query. This would be
provided as the LastEvaluatedKey in that query.
:type count: bool
:param count: If True, Amazon DynamoDB returns a total
number of items for the Query operation, even if the
operation has no matching items for the assigned filter.
If count is True, the actual items are not returned and
the count is accessible as the ``count`` attribute of
the returned object.
:type item_class: Class
:param item_class: Allows you to override the class used
to generate the items. This should be a subclass of
:class:`boto.dynamodb.item.Item`
"""
return self.layer2.query(self, hash_key, *args, **kw)
def scan(self, *args, **kw):
"""
Scan through this table, this is a very long
and expensive operation, and should be avoided if
at all possible.
:type scan_filter: A dict
:param scan_filter: A dictionary where the key is the
attribute name and the value is a
:class:`boto.dynamodb.condition.Condition` object.
Valid Condition objects include:
* EQ - equal (1)
* NE - not equal (1)
* LE - less than or equal (1)
* LT - less than (1)
* GE - greater than or equal (1)
* GT - greater than (1)
* NOT_NULL - attribute exists (0, use None)
* NULL - attribute does not exist (0, use None)
* CONTAINS - substring or value in list (1)
* NOT_CONTAINS - absence of substring or value in list (1)
* BEGINS_WITH - substring prefix (1)
* IN - exact match in list (N)
* BETWEEN - >= first value, <= second value (2)
:type attributes_to_get: list
:param attributes_to_get: A list of attribute names.
If supplied, only the specified attribute names will
be returned. Otherwise, all attributes will be returned.
:type request_limit: int
:param request_limit: The maximum number of items to retrieve
from Amazon DynamoDB on each request. You may want to set
a specific request_limit based on the provisioned throughput
of your table. The default behavior is to retrieve as many
results as possible per request.
:type max_results: int
:param max_results: The maximum number of results that will
be retrieved from Amazon DynamoDB in total. For example,
if you only wanted to see the first 100 results from the
query, regardless of how many were actually available, you
could set max_results to 100 and the generator returned
from the query method will only yeild 100 results max.
:type count: bool
:param count: If True, Amazon DynamoDB returns a total
number of items for the Scan operation, even if the
operation has no matching items for the assigned filter.
If count is True, the actual items are not returned and
the count is accessible as the ``count`` attribute of
the returned object.
:type exclusive_start_key: list or tuple
:param exclusive_start_key: Primary key of the item from
which to continue an earlier query. This would be
provided as the LastEvaluatedKey in that query.
:type item_class: Class
:param item_class: Allows you to override the class used
to generate the items. This should be a subclass of
:class:`boto.dynamodb.item.Item`
:return: A TableGenerator (generator) object which will iterate
over all results
:rtype: :class:`boto.dynamodb.layer2.TableGenerator`
"""
return self.layer2.scan(self, *args, **kw)
def batch_get_item(self, keys, attributes_to_get=None):
"""
Return a set of attributes for a multiple items from a single table
using their primary keys. This abstraction removes the 100 Items per
batch limitations as well as the "UnprocessedKeys" logic.
:type keys: list
:param keys: A list of scalar or tuple values. Each element in the
list represents one Item to retrieve. If the schema for the
table has both a HashKey and a RangeKey, each element in the
list should be a tuple consisting of (hash_key, range_key). If
the schema for the table contains only a HashKey, each element
in the list should be a scalar value of the appropriate type
for the table schema. NOTE: The maximum number of items that
can be retrieved for a single operation is 100. Also, the
number of items retrieved is constrained by a 1 MB size limit.
:type attributes_to_get: list
:param attributes_to_get: A list of attribute names.
If supplied, only the specified attribute names will
be returned. Otherwise, all attributes will be returned.
:return: A TableBatchGenerator (generator) object which will
iterate over all results
:rtype: :class:`boto.dynamodb.table.TableBatchGenerator`
"""
return TableBatchGenerator(self, keys, attributes_to_get)
EliteHackz.ORG
Revonzy Mini Shell
root@revonzy.com