# Copyright (C) 2019 OpenIO SAS, as part of OpenIO SDS
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from cliff import lister
from oio.cli.admin.common import AccountCommandMixin, ContainerCommandMixin, \
ObjectCommandMixin, ChunkCommandMixin
from oio.crawler.integrity import Checker, Target, DEFAULT_DEPTH
[docs]class ItemCheckCommand(lister.Lister):
"""
Various parameters that apply to all check commands.
"""
columns = ('Type', 'Item', 'Status', 'Errors')
success = True
checker = None
@property
def logger(self):
return self.app.client_manager.logger
[docs] def get_parser(self, prog_name):
parser = super(ItemCheckCommand, self).get_parser(prog_name)
parser.add_argument(
'--attempts',
type=int,
default=1,
help="Number of attempts for listing requests (default: 1)."
)
parser.add_argument(
'--checksum',
action='store_true',
help=("Perform checksum comparisons.")
)
parser.add_argument(
'--concurrency', '--workers', type=int,
default=30,
help="Number of concurrent checks (default: 30)."
)
parser.add_argument(
'-o', '--output',
help=("Output file. Will contain elements in error. "
"Can later be passed to stdin of the legacy "
"oio-crawler-integrity to re-check only these elements.")
)
parser.add_argument(
'--output-for-chunk-rebuild',
help=("Write chunk errors in a file with a format "
"suitable as 'openio-admin chunk rebuild' input.")
)
return parser
def _take_action(self, parsed_args):
raise NotImplementedError()
[docs] def take_action(self, parsed_args):
self.logger.debug('take_action(%s)', parsed_args)
self.checker = Checker(
self.app.options.ns,
concurrency=parsed_args.concurrency,
error_file=parsed_args.output,
rebuild_file=parsed_args.output_for_chunk_rebuild,
request_attempts=parsed_args.attempts,
check_hash=parsed_args.checksum,
logger=self.logger)
return self.columns, self._take_action(parsed_args)
def _format_results(self):
for res in self.checker.run():
if not res.has_errors:
status = 'OK'
yield (res.type, repr(res), status, str(None))
else:
self.success = False
status = 'error'
yield (res.type, repr(res),
status, res.latest_error_result().errors_to_str())
[docs] def run(self, parsed_args):
super(ItemCheckCommand, self).run(parsed_args)
if not self.success:
return 1
[docs]class RecursiveCheckCommand(ItemCheckCommand):
"""ItemCheckCommand with additional parameters to control recursion."""
[docs] def get_parser(self, prog_name):
parser = super(RecursiveCheckCommand, self).get_parser(prog_name)
parser.add_argument(
'--depth', '--max-depth',
type=int,
default=DEFAULT_DEPTH,
help=("How deep to recurse. 0 means do not recurse. "
"N > 0 means recurse N levels below the specified item type "
"(namespace -> account -> container -> object -> chunk, "
"default: %d)." % DEFAULT_DEPTH)
)
return parser
[docs]class AccountCheck(AccountCommandMixin, RecursiveCheckCommand):
"""
Check an account for problems.
"""
[docs] def get_parser(self, prog_name):
parser = super(AccountCheck, self).get_parser(prog_name)
AccountCommandMixin.patch_parser(self, parser)
return parser
def _take_action(self, parsed_args):
for acct in parsed_args.accounts:
target = Target(acct)
self.checker.check(target, parsed_args.depth)
return self._format_results()
[docs] def take_action(self, parsed_args):
AccountCommandMixin.check_and_load_parsed_args(
self, self.app, parsed_args)
return super(AccountCheck, self).take_action(parsed_args)
[docs]class ContainerCheck(ContainerCommandMixin, RecursiveCheckCommand):
"""
Check a container for problems.
Quick checks on the account owning
the container will also be performed.
"""
[docs] def get_parser(self, prog_name):
parser = super(ContainerCheck, self).get_parser(prog_name)
ContainerCommandMixin.patch_parser(self, parser)
return parser
def _take_action(self, parsed_args):
containers = self.resolve_containers(self.app, parsed_args, no_id=True)
for account, container_name, _ in containers:
target = Target(account, container_name)
self.checker.check(target, parsed_args.depth)
return self._format_results()
[docs] def take_action(self, parsed_args):
ContainerCommandMixin.check_and_load_parsed_args(
self, self.app, parsed_args)
return super(ContainerCheck, self).take_action(parsed_args)
[docs]class ObjectCheck(ObjectCommandMixin, RecursiveCheckCommand):
"""
Check an object for problems.
Quick checks on the account and the container
owning the object will also be performed.
"""
[docs] def get_parser(self, prog_name):
parser = super(ObjectCheck, self).get_parser(prog_name)
ObjectCommandMixin.patch_parser(self, parser)
return parser
def _take_action(self, parsed_args):
account, _, objects = self.resolve_objects(self.app, parsed_args)
for container, obj_name, version in objects:
target = Target(account, container, obj_name, version=version)
self.checker.check(target, parsed_args.depth)
return self._format_results()
[docs] def take_action(self, parsed_args):
ObjectCommandMixin.check_and_load_parsed_args(
self, self.app, parsed_args)
return super(ObjectCheck, self).take_action(parsed_args)
[docs]class ChunkCheck(ChunkCommandMixin, ItemCheckCommand):
"""
Check a chunk for problems.
Quick checks on the account, the container
and the object owning the chunk will also be performed.
"""
[docs] def get_parser(self, prog_name):
parser = super(ChunkCheck, self).get_parser(prog_name)
ChunkCommandMixin.patch_parser(self, parser)
return parser
def _take_action(self, parsed_args):
for chunk in parsed_args.chunks:
target = Target(self.app.options.account, chunk=chunk)
self.checker.check(target)
return self._format_results()
[docs] def take_action(self, parsed_args):
ChunkCommandMixin.check_and_load_parsed_args(
self, self.app, parsed_args)
return super(ChunkCheck, self).take_action(parsed_args)