void HierarchicalAllocatorProcess::allocate(
const hashset<SlaveID>& slaveIds_)
{
++metrics.allocation_runs;
// Compute the offerable resources, per framework:
// (1) For reserved resources on the slave, allocate these to a
// framework having the corresponding role.
// (2) For unreserved resources on the slave, allocate these
// to a framework of any role.
hashmap<FrameworkID, hashmap<SlaveID, Resources>> offerable;
// NOTE: This function can operate on a small subset of slaves, we have to
// make sure that we don't assume cluster knowledge when summing resources
// from that set.
vector<SlaveID> slaveIds;
slaveIds.reserve(slaveIds_.size());
// Filter out non-whitelisted and deactivated slaves in order not to send
// offers for them.
foreach (const SlaveID& slaveId, slaveIds_) {
if (isWhitelisted(slaveId) && slaves[slaveId].activated) {
slaveIds.push_back(slaveId);
}
}
// Randomize the order in which slaves' resources are allocated.
//
// TODO(vinod): Implement a smarter sorting algorithm.
std::random_shuffle(slaveIds.begin(), slaveIds.end());
// Returns the __quantity__ of resources allocated to a quota role. Since we
// account for reservations and persistent volumes toward quota, we strip
// reservation and persistent volume related information for comparability.
// The result is used to determine whether a role's quota is satisfied, and
// also to determine how many resources the role would need in order to meet
// its quota.
//
// NOTE: Revocable resources are excluded in `quotaRoleSorter`.
auto getQuotaRoleAllocatedResources = [this](const
string& role) {
CHECK(quotas.contains(role));
// NOTE: `allocationScalarQuantities` omits dynamic reservation and
// persistent volume info, but we additionally strip `role` here.
Resources resources;
foreach (Resource resource,
quotaRoleSorter->allocationScalarQuantities(role)) {
CHECK(!resource.has_reservation());
CHECK(!resource.has_disk());
resource.set_role("*");
resources += resource;
}
return resources;
};
// Quota comes first and fair share second. Here we process only those
// roles, for which quota is set (quota'ed roles). Such roles form a
// special allocation group with a dedicated sorter.
foreach (const SlaveID& slaveId, slaveIds) {
foreach (const
string& role, quotaRoleSorter->sort()) {
CHECK(quotas.contains(role));
// If there are no active frameworks in this role, we do not
// need to do any allocations for this role.
if (!activeRoles.contains(role)) {
continue;
}
// Get the total quantity of resources allocated to a quota role. The
// value omits role, reservation, and persistence info.
Resources roleConsumedResources = getQuotaRoleAllocatedResources(role);
// If quota for the role is satisfied, we do not need to do any further
// allocations for this role, at least at this stage.
//
// TODO(alexr): Skipping satisfied roles is pessimistic. Better
// alternatives are:
// * A custom sorter that is aware of quotas and sorts accordingly.
// * Removing satisfied roles from the sorter.
if (roleConsumedResources.contains(quotas[role].info.guarantee())) {
continue;
}
// Fetch frameworks according to their fair share.
foreach (const
string& frameworkId_, frameworkSorters[role]->sort()) {
FrameworkID frameworkId;
frameworkId.set_value(frameworkId_);
// If the framework has suppressed offers, ignore. The unallocated
// part of the quota will not be allocated to other roles.
if (frameworks[frameworkId].suppressed) {
continue;
}
// Only offer resources from slaves that have GPUs to
// frameworks that are capable of receiving GPUs.
// See MESOS-5634.
if (!frameworks[frameworkId].gpuAware &&
slaves[slaveId].total.gpus().getOrElse(0) > 0) {
continue;
}
// Calculate the currently available resources on the slave.
Resources available = slaves[slaveId].total - slaves[slaveId].allocated;
// The resources we offer are the unreserved resources as well as the
// reserved resources for this particular role. This is necessary to
// ensure that we don't offer resources that are reserved for another
// role.
//
// NOTE: Currently, frameworks are allowed to have '*' role.
// Calling reserved('*') returns an empty Resources object.
//
// Quota is satisfied from the available non-revocable resources on the
// agent. It's important that we include reserved resources here since
// reserved resources are accounted towards the quota guarantee. If we
// were to rely on stage 2 to offer them out, they would not be checked
// against the quota guarantee.
Resources resources =
(available.unreserved() + available.reserved(role)).nonRevocable();
// It is safe to break here, because all frameworks under a role would
// consider the same resources, so in case we don't have allocatable
// resources, we don't have to check for other frameworks under the
// same role. We only break out of the innermost loop, so the next step
// will use the same `slaveId`, but a different role.
//
// NOTE: The resources may not be allocatable here, but they can be
// accepted by one of the frameworks during the second allocation
// stage.
if (!allocatable(resources)) {
break;
}
// If the framework filters these resources, ignore. The unallocated
// part of the quota will not be allocated to other roles.
if (isFiltered(frameworkId, slaveId, resources)) {
continue;
}
VLOG(2) << "Allocating " << resources << " on agent " << slaveId
<< " to framework " << frameworkId
<< " as part of its role quota";
// NOTE: We perform "coarse-grained" allocation for quota'ed
// resources, which may lead to overcommitment of resources beyond
// quota. This is fine since quota currently represents a guarantee.
offerable[frameworkId][slaveId] += resources;
slaves[slaveId].allocated += resources;
// Resources allocated as part of the quota count towards the
// role's and the framework's fair share.
//
// NOTE: Revocable resources have already been excluded.
frameworkSorters[role]->add(slaveId, resources);
frameworkSorters[role]->allocated(frameworkId_, slaveId, resources);
roleSorter->allocated(role, slaveId, resources);
quotaRoleSorter->allocated(role, slaveId, resources);
}
}
}
// Calculate the total quantity of scalar resources (including revocable
// and reserved) that are available for allocation in the next round. We
// need this in order to ensure we do not over-allocate resources during
// the second stage.
//
// For performance reasons (MESOS-4833), this omits information about
// dynamic reservations or persistent volumes in the resources.
//
// NOTE: We use total cluster resources, and not just those based on the
// agents participating in the current allocation (i.e. provided as an
// argument to the `allocate()` call) so that frameworks in roles without
// quota are not unnecessarily deprived of resources.
Resources remainingClusterResources = roleSorter->totalScalarQuantities();
foreachkey (const
string& role, activeRoles) {
remainingClusterResources -= roleSorter->allocationScalarQuantities(role);
}
// Frameworks in a quota'ed role may temporarily reject resources by
// filtering or suppressing offers. Hence quotas may not be fully allocated.
Resources unallocatedQuotaResources;
foreachpair (const
string& name, const Quota& quota, quotas) {
// Compute the amount of quota that the role does not have allocated.
//
// NOTE: Revocable resources are excluded in `quotaRoleSorter`.
// NOTE: Only scalars are considered for quota.
Resources allocated = getQuotaRoleAllocatedResources(name);
const Resources required = quota.info.guarantee();
unallocatedQuotaResources += (required - allocated);
}
// Determine how many resources we may allocate during the next stage.
//
// NOTE: Resources for quota allocations are already accounted in
// `remainingClusterResources`.
remainingClusterResources -= unallocatedQuotaResources;
// To ensure we do not over-allocate resources during the second stage
// with all frameworks, we use 2 stopping criteria:
// * No available resources for the second stage left, i.e.
// `remainingClusterResources` - `allocatedStage2` is empty.
// * A potential offer will force the second stage to use more resources
// than available, i.e. `remainingClusterResources` does not contain
// (`allocatedStage2` + potential offer). In this case we skip this
// agent and continue to the next one.
//
// NOTE: Like `remainingClusterResources`, `allocatedStage2` omits
// information about dynamic reservations and persistent volumes for
// performance reasons. This invariant is preserved because we only add
// resources to it that have also had this metadata stripped from them
// (typically by using `Resources::createStrippedScalarQuantity`).
Resources allocatedStage2;
// At this point resources for quotas are allocated or accounted for.
// Proceed with allocating the remaining free pool.
foreach (const SlaveID& slaveId, slaveIds) {
// If there are no resources available for the second stage, stop.
if (!allocatable(remainingClusterResources - allocatedStage2)) {
break;
}
foreach (const
string& role, roleSorter->sort()) {
foreach (const
string& frameworkId_,
frameworkSorters[role]->sort()) {
FrameworkID frameworkId;
frameworkId.set_value(frameworkId_);
// If the framework has suppressed offers, ignore.
if (frameworks[frameworkId].suppressed) {
continue;
}
// Only offer resources from slaves that have GPUs to
// frameworks that are capable of receiving GPUs.
// See MESOS-5634.
if (!frameworks[frameworkId].gpuAware &&
slaves[slaveId].total.gpus().getOrElse(0) > 0) {
continue;
}
// Calculate the currently available resources on the slave.
Resources available = slaves[slaveId].total - slaves[slaveId].allocated;
// The resources we offer are the unreserved resources as well as the
// reserved resources for this particular role. This is necessary to
// ensure that we don't offer resources that are reserved for another
// role.
//
// NOTE: Currently, frameworks are allowed to have '*' role.
// Calling reserved('*') returns an empty Resources object.
//
// NOTE: We do not offer roles with quota any more non-revocable
// resources once their quota is satisfied. However, note that this is
// not strictly true due to the coarse-grained nature (per agent) of the
// allocation algorithm in stage 1.
//
// TODO(mpark): Offer unreserved resources as revocable beyond quota.
Resources resources = available.reserved(role);
if (!quotas.contains(role)) {
resources += available.unreserved();
}
// It is safe to break here, because all frameworks under a role would
// consider the same resources, so in case we don't have allocatable
// resources, we don't have to check for other frameworks under the
// same role. We only break out of the innermost loop, so the next step
// will use the same slaveId, but a different role.
//
// The difference to the second `allocatable` check is that here we also
// check for revocable resources, which can be disabled on a per frame-
// work basis, which requires us to go through all frameworks in case we
// have allocatable revocable resources.
if (!allocatable(resources)) {
break;
}
// Remove revocable resources if the framework has not opted for them.
if (!frameworks[frameworkId].revocable) {
resources = resources.nonRevocable();
}
// If the resources are not allocatable, ignore. We can not break
// here, because another framework under the same role could accept
// revocable resources and breaking would skip all other frameworks.
if (!allocatable(resources)) {
continue;
}
// If the framework filters these resources, ignore.
if (isFiltered(frameworkId, slaveId, resources)) {
continue;
}
// If the offer generated by `resources` would force the second
// stage to use more than `remainingClusterResources`, move along.
// We do not terminate early, as offers generated further in the
// loop may be small enough to fit within `remainingClusterResources`.
const Resources scalarQuantity =
resources.createStrippedScalarQuantity();
if (!remainingClusterResources.contains(
allocatedStage2 + scalarQuantity)) {
continue;
}
VLOG(2) << "Allocating " << resources << " on agent " << slaveId
<< " to framework " << frameworkId;
// NOTE: We perform "coarse-grained" allocation, meaning that we always
// allocate the entire remaining slave resources to a single framework.
//
// NOTE: We may have already allocated some resources on the current
// agent as part of quota.
offerable[frameworkId][slaveId] += resources;
allocatedStage2 += scalarQuantity;
slaves[slaveId].allocated += resources;
frameworkSorters[role]->add(slaveId, resources);
frameworkSorters[role]->allocated(frameworkId_, slaveId, resources);
roleSorter->allocated(role, slaveId, resources);
if (quotas.contains(role)) {
// See comment at `quotaRoleSorter` declaration regarding
// non-revocable.
quotaRoleSorter->allocated(role, slaveId, resources.nonRevocable());
}
}
}
}
if (offerable.empty()) {
VLOG(1) << "No allocations performed";
} else {
// Now offer the resources to each framework.
foreachkey (const FrameworkID& frameworkId, offerable) {
offerCallback(frameworkId, offerable[frameworkId]);
}
}
// NOTE: For now, we implement maintenance inverse offers within the
// allocator. We leverage the existing timer/cycle of offers to also do any
// "deallocation" (inverse offers) necessary to satisfy maintenance needs.
deallocate(slaveIds_);
}
