oio.account package

Submodules

oio.account.backend module

class oio.account.backend.AccountBackend(conf)[source]

Bases: oio.common.redis_conn.RedisConnection

akey(account)[source]

Build the key of an account description

bkey(bucket)[source]

Build the key of a bucket description

blistkey(account)[source]

Build the key of an account’s bucket list

buckets_pattern = <_sre.SRE_Pattern object at 0x277aab0>
cast_fields(info)[source]

Cast dict entries to the type they are supposed to be.

static ckey(account, name)[source]

Build the key of a container description

clistkey(account)[source]

Build the key of an account’s container list

create_account(**kwargs)[source]
delete_account(**kwargs)[source]
flush_account(**kwargs)[source]
get_account_metadata(**kwargs)[source]
get_bucket_info(**kwargs)[source]

Get all available information about a bucket.

get_container_info(**kwargs)[source]

Get all available information about a container, including some information coming from the bucket it belongs to.

info_account(**kwargs)[source]
list_accounts(**kwargs)[source]

Get the list of all accounts.

list_buckets(**kwargs)[source]

Get the list of buckets of the specified account.

Returns:the list of buckets (with metadata), and the next marker (in case the list is truncated).
list_containers(**kwargs)[source]
lua_flush_account = "\n local account_id = redis.call('HGET', KEYS[1], 'id');\n if not account_id then\n return redis.error_reply('no_account');\n end;\n\n redis.call('HMSET', KEYS[1], 'objects', 0, 'bytes', 0,\n 'damaged_objects', 0, 'missing_chunks', 0)\n\n local containers = redis.call('ZRANGE', KEYS[2], 0, -1);\n redis.call('DEL', KEYS[2]);\n\n for _,container in ipairs(containers) do\n redis.call('DEL', KEYS[3] .. container);\n end;\n "
lua_get_extended_container_info = "\n local ckey = KEYS[1]\n local bucket_prefix = ARGV[1]\n local bname = redis.call('HGET', ckey, 'bucket')\n\n local replication_enabled = 'false'\n if bname then\n local bkey = bucket_prefix .. bname\n local enabled_str = redis.call('HGET', bkey, 'replication_enabled')\n if enabled_str ~= nil then\n -- Do not cast into boolean here\n replication_enabled = enabled_str\n end\n end\n local res = redis.call('HGETALL', ckey)\n table.insert(res, 'replication_enabled')\n table.insert(res, replication_enabled)\n return res\n "
lua_is_sup = '\n -- With lua float we are losing precision for this reason\n -- we keep the number as a string\n local is_sup = function(a,b)\n local int_a = string.match(a,"%d+")\n local int_b = string.match(b,"%d+")\n if string.len(int_a) > string.len(int_b) then\n return true;\n end;\n return a > b;\n end;\n '
lua_refresh_account = "\n local account_id = redis.call('HGET', KEYS[1], 'id');\n if not account_id then\n return redis.error_reply('no_account');\n end;\n\n local containers = redis.call('ZRANGE', KEYS[2], 0, -1);\n local container_key = ''\n local bytes_sum = 0;\n local objects_sum = 0;\n local damaged_objects_sum = 0;\n local missing_chunks_sum = 0;\n for _,container in ipairs(containers) do\n container_key = KEYS[3] .. container;\n objects_sum = objects_sum + redis.call('HGET', container_key,\n 'objects')\n bytes_sum = bytes_sum + redis.call('HGET', container_key, 'bytes')\n local damaged_objects = redis.call('HGET', container_key,\n 'damaged_objects')\n if damaged_objects == false then\n damaged_objects = 0\n end\n damaged_objects_sum = damaged_objects_sum + damaged_objects\n local missing_chunks = redis.call('HGET', container_key,\n 'missing_chunks')\n if missing_chunks == false then\n missing_chunks = 0\n end\n missing_chunks_sum = missing_chunks_sum + missing_chunks\n end;\n\n redis.call('HMSET', KEYS[1], 'objects', objects_sum,\n 'bytes', bytes_sum,\n 'damaged_objects', damaged_objects_sum,\n 'missing_chunks', missing_chunks_sum)\n "
lua_update_bucket_func = "\n -- Note that bucket is the key name, not just the bucket name\n -- (it has a prefix).\n local update_bucket_stats = function(\n bucket, account, mtime,\n inc_objects, inc_bytes, inc_damaged_objects, inc_missing_chunks)\n -- Set the bucket owner.\n -- FIXME(FVE): do some checks instead of overwriting\n redis.call('HSET', bucket, 'account', account)\n\n -- Increment the counters.\n redis.call('HINCRBY', bucket, 'objects', inc_objects)\n redis.call('HINCRBY', bucket, 'bytes', inc_bytes)\n redis.call('HINCRBY', bucket, 'damaged_objects', inc_damaged_objects)\n redis.call('HINCRBY', bucket, 'missing_chunks', inc_missing_chunks)\n\n -- Finally update the modification time.\n redis.call('HSET', bucket, 'mtime', mtime)\n end\n "
lua_update_bucket_list = "\n -- Key to the set of bucket of a specific account\n local bucket_set = KEYS[1]\n -- Actual name of the bucket\n local bucket_name = ARGV[1]\n -- True if the bucket has just been deleted\n local deleted = ARGV[2]\n\n if deleted ~= 'True' then\n redis.call('ZADD', bucket_set, 0, bucket_name);\n else\n redis.call('ZREM', bucket_set, bucket_name);\n end\n "
lua_update_container = '\n -- With lua float we are losing precision for this reason\n -- we keep the number as a string\n local is_sup = function(a,b)\n local int_a = string.match(a,"%d+")\n local int_b = string.match(b,"%d+")\n if string.len(int_a) > string.len(int_b) then\n return true;\n end;\n return a > b;\n end;\n \n -- Note that bucket is the key name, not just the bucket name\n -- (it has a prefix).\n local update_bucket_stats = function(\n bucket, account, mtime,\n inc_objects, inc_bytes, inc_damaged_objects, inc_missing_chunks)\n -- Set the bucket owner.\n -- FIXME(FVE): do some checks instead of overwriting\n redis.call(\'HSET\', bucket, \'account\', account)\n\n -- Increment the counters.\n redis.call(\'HINCRBY\', bucket, \'objects\', inc_objects)\n redis.call(\'HINCRBY\', bucket, \'bytes\', inc_bytes)\n redis.call(\'HINCRBY\', bucket, \'damaged_objects\', inc_damaged_objects)\n redis.call(\'HINCRBY\', bucket, \'missing_chunks\', inc_missing_chunks)\n\n -- Finally update the modification time.\n redis.call(\'HSET\', bucket, \'mtime\', mtime)\n end\n \n -- KEYS[1] account name\n -- KEYS[2] key to the container hash\n -- KEYS[3] key to the account\'s container set\n -- KEYS[4] key to the account hash\n -- KEYS[5] key to the bucket hash\n local bkey = KEYS[5]\n -- ARGV[1] container name\n -- ARGV[2] mtime\n -- ARGV[3] dtime\n -- ARGV[4] new object count\n -- ARGV[5] new total size\n -- ARGV[6] damaged objects\n -- ARGV[7] missing chunks\n -- ARGV[8] autocreate account?\n -- ARGV[9] current timestamp\n local now = ARGV[9]\n -- ARGV[10] container key expiration time\n -- ARGV[11] autocreate container?\n -- ARGV[12] update bucket object count?\n local update_bucket_object_count = ARGV[12]\n\n local account_id = redis.call(\'HGET\', KEYS[4], \'id\');\n if not account_id then\n if ARGV[8] == \'True\' then\n redis.call(\'HSET\', \'accounts:\', KEYS[1], 1);\n redis.call(\'HMSET\', KEYS[4], \'id\', KEYS[1],\n \'bytes\', 0, \'objects\', 0,\n \'damaged_objects\', 0, \'missing_chunks\', 0,\n \'ctime\', now);\n account_id = KEYS[1]\n else\n return redis.error_reply(\'no_account\');\n end;\n end;\n\n if ARGV[11] == \'False\' then\n local container_name = redis.call(\'HGET\', KEYS[2], \'name\');\n if not container_name then\n return redis.error_reply(\'no_container\');\n end;\n end;\n\n local name = ARGV[1];\n local mtime = redis.call(\'HGET\', KEYS[2], \'mtime\');\n local dtime = redis.call(\'HGET\', KEYS[2], \'dtime\');\n local objects = redis.call(\'HGET\', KEYS[2], \'objects\');\n local bytes = redis.call(\'HGET\', KEYS[2], \'bytes\');\n local damaged_objects = redis.call(\'HGET\', KEYS[2],\n \'damaged_objects\');\n local missing_chunks = redis.call(\'HGET\', KEYS[2],\n \'missing_chunks\');\n\n -- When the keys do not exist redis return false and not nil\n if dtime == false then\n dtime = \'0\'\n end\n if mtime == false then\n mtime = \'0\'\n end\n if objects == false then\n objects = 0\n else\n objects = tonumber(objects)\n end\n if bytes == false then\n bytes = 0\n else\n bytes = tonumber(bytes)\n end\n if damaged_objects == false then\n damaged_objects = 0\n else\n damaged_objects = tonumber(damaged_objects)\n end\n if missing_chunks == false then\n missing_chunks = 0\n else\n missing_chunks = tonumber(missing_chunks)\n end\n\n if ARGV[11] == \'False\' and is_sup(dtime, mtime) then\n return redis.error_reply(\'no_container\');\n end;\n\n local old_mtime = mtime;\n local inc_objects = 0;\n local inc_bytes = 0;\n local inc_damaged_objects = 0;\n local inc_missing_chunks = 0;\n\n if not is_sup(ARGV[3],dtime) and not is_sup(ARGV[2],mtime) then\n return redis.error_reply(\'no_update_needed\');\n end;\n\n if is_sup(ARGV[2],mtime) then\n mtime = ARGV[2];\n end;\n\n if is_sup(ARGV[3],dtime) then\n dtime = ARGV[3];\n end;\n if is_sup(dtime,mtime) then\n -- Protect against "minus zero".\n if objects ~= 0 then\n inc_objects = -objects;\n end\n if bytes ~= 0 then\n inc_bytes = -bytes;\n end\n if damaged_objects ~= 0 then\n inc_damaged_objects = -damaged_objects\n end\n if missing_chunks ~= 0 then\n inc_missing_chunks = -missing_chunks;\n end\n redis.call(\'HMSET\', KEYS[2],\n \'bytes\', 0, \'objects\', 0,\n \'damaged_objects\', 0, \'missing_chunks\', 0);\n redis.call(\'EXPIRE\', KEYS[2], tonumber(ARGV[10]));\n redis.call(\'ZREM\', KEYS[3], name);\n elseif is_sup(mtime,old_mtime) then\n redis.call(\'PERSIST\', KEYS[2]);\n inc_objects = tonumber(ARGV[4]) - objects\n inc_bytes = tonumber(ARGV[5]) - bytes\n inc_damaged_objects = tonumber(ARGV[6]) - damaged_objects\n inc_missing_chunks = tonumber(ARGV[7]) - missing_chunks\n redis.call(\'HMSET\', KEYS[2],\n \'objects\', tonumber(ARGV[4]),\n \'bytes\', tonumber(ARGV[5]),\n \'damaged_objects\', tonumber(ARGV[6]),\n \'missing_chunks\', tonumber(ARGV[7]));\n redis.call(\'ZADD\', KEYS[3], \'0\', name);\n else\n return redis.error_reply(\'no_update_needed\');\n end;\n\n redis.call(\'HMSET\', KEYS[2], \'mtime\', mtime,\n \'dtime\', dtime, \'name\', name)\n if inc_objects ~= 0 then\n redis.call(\'HINCRBY\', KEYS[4], \'objects\', inc_objects);\n end;\n if inc_bytes ~= 0 then\n redis.call(\'HINCRBY\', KEYS[4], \'bytes\', inc_bytes);\n end;\n if inc_damaged_objects ~= 0 then\n redis.call(\'HINCRBY\', KEYS[4], \'damaged_objects\',\n inc_damaged_objects);\n end;\n if inc_missing_chunks ~= 0 then\n redis.call(\'HINCRBY\', KEYS[4], \'missing_chunks\',\n inc_missing_chunks);\n end;\n\n if bkey ~= \'False\' then\n -- For container holding MPU segments, we do not want to count\n -- each segment as an object. But we still want to consider\n -- their size.\n if update_bucket_object_count ~= \'True\' then\n inc_objects = 0\n end\n update_bucket_stats(bkey, account_id, now,\n inc_objects, inc_bytes,\n inc_damaged_objects, inc_missing_chunks)\n end\n '
refresh_account(**kwargs)[source]
status(**kwargs)[source]
update_account_metadata(**kwargs)[source]
update_bucket_metadata(**kwargs)[source]

Update (or delete) bucket metadata.

Parameters:
  • metadata – dict of entries to set (or update)
  • to_delete – iterable of keys to delete
update_container(**kwargs)[source]

oio.account.client module

class oio.account.client.AccountClient(conf, endpoint=None, proxy_endpoint=None, refresh_delay=3600.0, logger=None, **kwargs)[source]

Bases: oio.api.base.HttpApi

Simple client API for the account service.

account_create(account, **kwargs)[source]

Create an account.

Parameters:account (str) – name of the account to create
Returns:True if the account has been created
account_delete(account, **kwargs)[source]

Delete an account.

Parameters:account (str) – name of the account to delete
account_flush(account, **kwargs)[source]

Flush all containers of an account

Parameters:account (str) – name of the account to flush
account_list(**kwargs)[source]

List accounts.

account_refresh(account, **kwargs)[source]

Refresh counters of an account

Parameters:account (str) – name of the account to refresh
account_request(*args, **kwargs)[source]

Make a request to the account service.

account_show(account, **kwargs)[source]

Get information about an account.

account_update(account, metadata, to_delete, **kwargs)[source]

Update metadata of the specified account.

Parameters:
  • metadata (dict) – dictionary of properties that must be set or updated.
  • to_delete (list) – list of property keys that must be removed.
bucket_list(account, limit=None, marker=None, prefix=None, **kwargs)[source]

Get the list of buckets of an account.

Parameters:
  • account (str) – account from which to get the bucket list
  • limit (int) – maximum number of results to return
  • marker (str) – name of the bucket from where to start the listing
  • prefix
Return type:

dict with ‘ctime’ (float), ‘buckets’ (int), ‘bytes’ (int), ‘objects’ (int), ‘containers’ (int), ‘id’ (str), ‘metadata’ (dict), ‘listing’ (list), ‘truncated’ and ‘next_marker’. ‘listing’ contains dicts of container metadata (name, number of objects, number of bytes and modification time).

bucket_show(bucket, **kwargs)[source]

Get information about a bucket.

bucket_update(bucket, metadata, to_delete, **kwargs)[source]

Update metadata of the specified bucket.

Parameters:
  • metadata (dict) – dictionary of properties that must be set or updated.
  • to_delete (list) – list of property keys that must be removed.
container_list(account, limit=None, marker=None, end_marker=None, prefix=None, delimiter=None, s3_buckets_only=False, **kwargs)[source]

Get the list of containers of an account.

Parameters:
  • account (str) – account from which to get the container list
  • limit (int) – maximum number of results to return
  • marker (str) – name of the container from where to start the listing
  • end_marker
  • prefix
  • delimiter
  • s3_buckets_only (bool) – list only S3 buckets.
Return type:

dict with ‘ctime’ (float), ‘bytes’ (int), ‘objects’ (int), ‘containers’ (int), ‘id’ (str), ‘metadata’ (dict) and ‘listing’ (list). ‘listing’ contains lists of container metadata (name, number of objects, number of bytes, whether it is a prefix, and modification time).

container_reset(account, container, mtime, **kwargs)[source]

Reset container of an account

Parameters:
  • account (str) – name of the account
  • container (str) – name of the container to reset
  • mtime – time of the modification
container_show(account, container, **kwargs)[source]

Get information about a container.

container_update(account, container, metadata=None, **kwargs)[source]

Update account with container-related metadata.

Parameters:
  • account (str) – name of the account to update
  • container (str) – name of the container whose metadata has changed
  • metadata – container metadata (“bytes”, “objects”,

“mtime”, “dtime”) :type metadata: dict

oio.account.rebuilder module

class oio.account.rebuilder.AccountRebuilder(conf, accounts=None, **kwargs)[source]

Bases: oio.common.tool.Tool

Rebuild the account services.

create_worker(queue_workers, queue_reply)[source]

Create worker to process the items.

run()[source]

Start processing all found items.

static string_from_item(item)[source]
class oio.account.rebuilder.AccountRebuilderWorker(tool, queue_workers, queue_reply)[source]

Bases: oio.common.tool.ToolWorker

oio.account.server module

class oio.account.server.Account(conf, backend, logger=None)[source]

Bases: oio.common.wsgi.WerkzeugApp

on_account_buckets(req, *args, **kwargs)[source]
on_account_container_reset(req)[source]
on_account_container_show(req, *args, **kwargs)[source]
on_account_container_update(req)[source]
on_account_containers(req, *args, **kwargs)[source]
on_account_create(req, *args, **kwargs)[source]
on_account_delete(req, *args, **kwargs)[source]
on_account_flush(req, *args, **kwargs)[source]
on_account_list(req, *args, **kwargs)[source]
on_account_refresh(req, *args, **kwargs)[source]
on_account_show(req, *args, **kwargs)[source]
on_account_update(req)[source]
on_bucket_show(req, *args, **kwargs)[source]

Get all available information about a bucket.

on_bucket_update(req)[source]

Update (or delete) bucket metadata.

on_status(req)[source]
oio.account.server.access_log(func)[source]
oio.account.server.create_app(conf, **kwargs)[source]

Module contents