import errno
import fnmatch
import re
from collections import namedtuple, OrderedDict
from functools import wraps
from typing import Optional, Dict, Any, List, Union, Callable, Iterable
import six
import yaml
from ceph.deployment.hostspec import HostSpec
from ceph.deployment.utils import unwrap_ipv6
class ServiceSpecValidationError(Exception):
"""
Defining an exception here is a bit problematic, cause you cannot properly catch it,
if it was raised in a different mgr module.
"""
def __init__(self,
msg: str,
errno: int = -errno.EINVAL):
super(ServiceSpecValidationError, self).__init__(msg)
self.errno = errno
def assert_valid_host(name):
p = re.compile('^[a-zA-Z0-9-]+$')
try:
assert len(name) <= 250, 'name is too long (max 250 chars)'
for part in name.split('.'):
assert len(part) > 0, '.-delimited name component must not be empty'
assert len(part) <= 63, '.-delimited name component must not be more than 63 chars'
assert p.match(part), 'name component must include only a-z, 0-9, and -'
except AssertionError as e:
raise ServiceSpecValidationError(e)
def handle_type_error(method):
@wraps(method)
def inner(cls, *args, **kwargs):
try:
return method(cls, *args, **kwargs)
except (TypeError, AttributeError) as e:
error_msg = '{}: {}'.format(cls.__name__, e)
raise ServiceSpecValidationError(error_msg)
return inner
class HostPlacementSpec(namedtuple('HostPlacementSpec', ['hostname', 'network', 'name'])):
def __str__(self):
res = ''
res += self.hostname
if self.network:
res += ':' + self.network
if self.name:
res += '=' + self.name
return res
@classmethod
@handle_type_error
def from_json(cls, data):
if isinstance(data, str):
return cls.parse(data)
return cls(**data)
def to_json(self) -> str:
return str(self)
@classmethod
def parse(cls, host, require_network=True):
# type: (str, bool) -> HostPlacementSpec
"""
Split host into host, network, and (optional) daemon name parts. The network
part can be an IP, CIDR, or ceph addrvec like '[v2:1.2.3.4:3300,v1:1.2.3.4:6789]'.
e.g.,
"myhost"
"myhost=name"
"myhost:1.2.3.4"
"myhost:1.2.3.4=name"
"myhost:1.2.3.0/24"
"myhost:1.2.3.0/24=name"
"myhost:[v2:1.2.3.4:3000]=name"
"myhost:[v2:1.2.3.4:3000,v1:1.2.3.4:6789]=name"
"""
# Matches from start to : or = or until end of string
host_re = r'^(.*?)(:|=|$)'
# Matches from : to = or until end of string
ip_re = r':(.*?)(=|$)'
# Matches from = to end of string
name_re = r'=(.*?)$'
# assign defaults
host_spec = cls('', '', '')
match_host = re.search(host_re, host)
if match_host:
host_spec = host_spec._replace(hostname=match_host.group(1))
name_match = re.search(name_re, host)
if name_match:
host_spec = host_spec._replace(name=name_match.group(1))
ip_match = re.search(ip_re, host)
if ip_match:
host_spec = host_spec._replace(network=ip_match.group(1))
if not require_network:
return host_spec
from ipaddress import ip_network, ip_address
networks = list() # type: List[str]
network = host_spec.network
# in case we have [v2:1.2.3.4:3000,v1:1.2.3.4:6478]
if ',' in network:
networks = [x for x in network.split(',')]
else:
if network != '':
networks.append(network)
for network in networks:
# only if we have versioned network configs
if network.startswith('v') or network.startswith('[v'):
# if this is ipv6 we can't just simply split on ':' so do
# a split once and rsplit once to leave us with just ipv6 addr
network = network.split(':', 1)[1]
network = network.rsplit(':', 1)[0]
try:
# if subnets are defined, also verify the validity
if '/' in network:
ip_network(six.text_type(network))
else:
ip_address(unwrap_ipv6(network))
except ValueError as e:
# logging?
raise e
host_spec.validate()
return host_spec
def validate(self):
assert_valid_host(self.hostname)
class PlacementSpec(object):
"""
For APIs that need to specify a host subset
"""
def __init__(self,
label=None, # type: Optional[str]
hosts=None, # type: Union[List[str],List[HostPlacementSpec]]
count=None, # type: Optional[int]
host_pattern=None # type: Optional[str]
):
# type: (...) -> None
self.label = label
self.hosts = [] # type: List[HostPlacementSpec]
if hosts:
self.set_hosts(hosts)
self.count = count # type: Optional[int]
#: fnmatch patterns to select hosts. Can also be a single host.
self.host_pattern = host_pattern # type: Optional[str]
self.validate()
def is_empty(self):
return self.label is None and \
not self.hosts and \
not self.host_pattern and \
self.count is None
def __eq__(self, other):
if isinstance(other, PlacementSpec):
return self.label == other.label \
and self.hosts == other.hosts \
and self.count == other.count \
and self.host_pattern == other.host_pattern
return NotImplemented
def set_hosts(self, hosts):
# To backpopulate the .hosts attribute when using labels or count
# in the orchestrator backend.
if all([isinstance(host, HostPlacementSpec) for host in hosts]):
self.hosts = hosts # type: ignore
else:
self.hosts = [HostPlacementSpec.parse(x, require_network=False) # type: ignore
for x in hosts if x]
# deprecated
def filter_matching_hosts(self, _get_hosts_func: Callable) -> List[str]:
return self.filter_matching_hostspecs(_get_hosts_func(as_hostspec=True))
def filter_matching_hostspecs(self, hostspecs: Iterable[HostSpec]) -> List[str]:
if self.hosts:
all_hosts = [hs.hostname for hs in hostspecs]
return [h.hostname for h in self.hosts if h.hostname in all_hosts]
elif self.label:
return [hs.hostname for hs in hostspecs if self.label in hs.labels]
elif self.host_pattern:
all_hosts = [hs.hostname for hs in hostspecs]
return fnmatch.filter(all_hosts, self.host_pattern)
else:
# This should be caught by the validation but needs to be here for
# get_host_selection_size
return []
def get_host_selection_size(self, hostspecs: Iterable[HostSpec]):
if self.count:
return self.count
return len(self.filter_matching_hostspecs(hostspecs))
def pretty_str(self):
"""
>>> #doctest: +SKIP
... ps = PlacementSpec(...) # For all placement specs:
... PlacementSpec.from_string(ps.pretty_str()) == ps
"""
kv = []
if self.hosts:
kv.append(';'.join([str(h) for h in self.hosts]))
if self.count:
kv.append('count:%d' % self.count)
if self.label:
kv.append('label:%s' % self.label)
if self.host_pattern:
kv.append(self.host_pattern)
return ';'.join(kv)
def __repr__(self):
kv = []
if self.count:
kv.append('count=%d' % self.count)
if self.label:
kv.append('label=%s' % repr(self.label))
if self.hosts:
kv.append('hosts={!r}'.format(self.hosts))
if self.host_pattern:
kv.append('host_pattern={!r}'.format(self.host_pattern))
return "PlacementSpec(%s)" % ', '.join(kv)
@classmethod
@handle_type_error
def from_json(cls, data):
c = data.copy()
hosts = c.get('hosts', [])
if hosts:
c['hosts'] = []
for host in hosts:
c['hosts'].append(HostPlacementSpec.from_json(host))
_cls = cls(**c)
_cls.validate()
return _cls
def to_json(self):
r = {}
if self.label:
r['label'] = self.label
if self.hosts:
r['hosts'] = [host.to_json() for host in self.hosts]
if self.count:
r['count'] = self.count
if self.host_pattern:
r['host_pattern'] = self.host_pattern
return r
def validate(self):
if self.hosts and self.label:
# TODO: a less generic Exception
raise ServiceSpecValidationError('Host and label are mutually exclusive')
if self.count is not None and self.count <= 0:
raise ServiceSpecValidationError("num/count must be > 1")
if self.host_pattern and self.hosts:
raise ServiceSpecValidationError('cannot combine host patterns and hosts')
for h in self.hosts:
h.validate()
@classmethod
def from_string(cls, arg):
# type: (Optional[str]) -> PlacementSpec
"""
A single integer is parsed as a count:
>>> PlacementSpec.from_string('3')
PlacementSpec(count=3)
A list of names is parsed as host specifications:
>>> PlacementSpec.from_string('host1 host2')
PlacementSpec(hosts=[HostPlacementSpec(hostname='host1', network='', name=''), HostPlacemen\
tSpec(hostname='host2', network='', name='')])
You can also prefix the hosts with a count as follows:
>>> PlacementSpec.from_string('2 host1 host2')
PlacementSpec(count=2, hosts=[HostPlacementSpec(hostname='host1', network='', name=''), Hos\
tPlacementSpec(hostname='host2', network='', name='')])
You can specify labels using `label: