Source code for oio.rebuilder.meta2_rebuilder

# Copyright (C) 2018 OpenIO SAS, as part of OpenIO SDS
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.


from datetime import datetime

from oio.common.utils import cid_from_name
from oio.rebuilder.meta_rebuilder import MetaRebuilder, MetaRebuilderWorker


[docs]class Meta2Rebuilder(MetaRebuilder): def __init__(self, conf, logger, **kwargs): super(Meta2Rebuilder, self).__init__(conf, logger, **kwargs) def _create_worker(self, **kwargs): return Meta2RebuilderWorker(self.conf, self.logger, **kwargs) def _fill_queue(self, queue, **kwargs): if self._fill_queue_from_file(queue, **kwargs): return accounts = self.api.account_list() for account in accounts: containers = self._full_container_list(account) for container in containers: cid = cid_from_name(account, container[0]) queue.put(cid) def _get_report(self, start_time, end_time, passes, errors, waiting_time, rebuilder_time, elapsed, total_references_processed, info, **kwargs): return ('DONE ' 'started=%(start_time)s ' 'ended=%(end_time)s ' 'elapsed=%(elapsed).2f ' 'passes=%(passes)d ' 'errors=%(errors)d ' 'meta2_references=%(references)d %(rate).2f/s ' 'waiting_time=%(waiting_time).2f ' 'rebuilder_time=%(rebuilder_time).2f ' '(rebuilder: %(success_rate).2f%%)' % { 'start_time': datetime.fromtimestamp( int(start_time)).isoformat(), 'end_time': datetime.fromtimestamp( int(end_time)).isoformat(), 'elapsed': elapsed, 'passes': passes, 'errors': errors, 'references': total_references_processed, 'rate': total_references_processed / elapsed, 'rebuilder_time': rebuilder_time, 'waiting_time': waiting_time, 'success_rate': 100 * ((total_references_processed - errors) / float(total_references_processed or 1)) })
[docs]class Meta2RebuilderWorker(MetaRebuilderWorker): def __init__(self, conf, logger, max_attempts=5, **kwargs): super(Meta2RebuilderWorker, self).__init__(conf, logger, 'meta2', **kwargs) def _get_report(self, num, start_time, end_time, total_time, report_time, **kwargs): return ('RUN ' 'worker=%(num)d ' 'started=%(start_time)s ' 'passes=%(passes)d ' 'errors=%(errors)d ' 'meta2_references=%(references)d %(rate).2f/s ' 'waiting_time=%(waiting_time).2f ' 'rebuilder_time=%(rebuilder_time).2f ' 'total_time=%(total_time).2f ' '(rebuilder: %(success_rate).2f%%)' % { 'num': num, 'start_time': datetime.fromtimestamp( int(report_time)).isoformat(), 'passes': self.passes, 'errors': self.errors, 'references': self.total_items_processed, 'rate': self.passes / (end_time - report_time), 'waiting_time': self.waiting_time, 'rebuilder_time': self.rebuilder_time, 'total_time': (end_time - start_time), 'success_rate': 100 * ((self.total_items_processed - self.errors) / float(self.total_items_processed or 1)) })