Intel(R) Threading Building Blocks Doxygen Documentation
version 4.2.3
|
Work stealing task scheduler. More...
#include <scheduler.h>
Public Member Functions | |
bool | is_task_pool_published () const |
bool | is_local_task_pool_quiescent () const |
bool | is_quiescent_local_task_pool_empty () const |
bool | is_quiescent_local_task_pool_reset () const |
void | attach_mailbox (affinity_id id) |
void | init_stack_info () |
Sets up the data necessary for the stealing limiting heuristics. More... | |
bool | can_steal () |
Returns true if stealing is allowed. More... | |
void | publish_task_pool () |
Used by workers to enter the task pool. More... | |
void | leave_task_pool () |
Leave the task pool. More... | |
void | reset_task_pool_and_leave () |
Resets head and tail indices to 0, and leaves task pool. More... | |
task ** | lock_task_pool (arena_slot *victim_arena_slot) const |
Locks victim's task pool, and returns pointer to it. The pointer can be NULL. More... | |
void | unlock_task_pool (arena_slot *victim_arena_slot, task **victim_task_pool) const |
Unlocks victim's task pool. More... | |
void | acquire_task_pool () const |
Locks the local task pool. More... | |
void | release_task_pool () const |
Unlocks the local task pool. More... | |
task * | prepare_for_spawning (task *t) |
Checks if t is affinitized to another thread, and if so, bundles it as proxy. More... | |
void | commit_spawned_tasks (size_t new_tail) |
Makes newly spawned tasks visible to thieves. More... | |
void | commit_relocated_tasks (size_t new_tail) |
Makes relocated tasks visible to thieves and releases the local task pool. More... | |
task * | get_task (__TBB_ISOLATION_EXPR(isolation_tag isolation)) |
Get a task from the local pool. More... | |
task * | get_task (size_t T) |
Get a task from the local pool at specified location T. More... | |
task * | get_mailbox_task (__TBB_ISOLATION_EXPR(isolation_tag isolation)) |
Attempt to get a task from the mailbox. More... | |
task * | steal_task (__TBB_ISOLATION_EXPR(isolation_tag isolation)) |
Attempts to steal a task from a randomly chosen thread/scheduler. More... | |
task * | steal_task_from (__TBB_ISOLATION_ARG(arena_slot &victim_arena_slot, isolation_tag isolation)) |
Steal task from another scheduler's ready pool. More... | |
size_t | prepare_task_pool (size_t n) |
Makes sure that the task pool can accommodate at least n more elements. More... | |
bool | cleanup_master (bool blocking_terminate) |
Perform necessary cleanup when a master thread stops using TBB. More... | |
void | assert_task_pool_valid () const |
void | attach_arena (arena *, size_t index, bool is_master) |
void | nested_arena_entry (arena *, size_t) |
void | nested_arena_exit () |
void | wait_until_empty () |
void | spawn (task &first, task *&next) __TBB_override |
For internal use only. More... | |
void | spawn_root_and_wait (task &first, task *&next) __TBB_override |
For internal use only. More... | |
void | enqueue (task &, void *reserved) __TBB_override |
For internal use only. More... | |
void | local_spawn (task *first, task *&next) |
void | local_spawn_root_and_wait (task *first, task *&next) |
virtual void | local_wait_for_all (task &parent, task *child)=0 |
void | free_scheduler () |
Destroy and deallocate this scheduler object. More... | |
task & | allocate_task (size_t number_of_bytes, __TBB_CONTEXT_ARG(task *parent, task_group_context *context)) |
Allocate task object, either from the heap or a free list. More... | |
template<free_task_hint h> | |
void | free_task (task &t) |
Put task on free list. More... | |
void | deallocate_task (task &t) |
Return task object to the memory allocator. More... | |
bool | is_worker () const |
True if running on a worker thread, false otherwise. More... | |
bool | outermost_level () const |
True if the scheduler is on the outermost dispatch level. More... | |
bool | master_outermost_level () const |
True if the scheduler is on the outermost dispatch level in a master thread. More... | |
bool | worker_outermost_level () const |
True if the scheduler is on the outermost dispatch level in a worker thread. More... | |
unsigned | max_threads_in_arena () |
Returns the concurrency limit of the current arena. More... | |
virtual task * | receive_or_steal_task (__TBB_ISOLATION_ARG(__TBB_atomic reference_count &completion_ref_count, isolation_tag isolation))=0 |
Try getting a task from other threads (via mailbox, stealing, FIFO queue, orphans adoption). More... | |
void | free_nonlocal_small_task (task &t) |
Free a small task t that that was allocated by a different scheduler. More... | |
![]() | |
virtual void | wait_for_all (task &parent, task *child)=0 |
For internal use only. More... | |
virtual | ~scheduler ()=0 |
Pure virtual destructor;. More... | |
Static Public Member Functions | |
static bool | is_version_3_task (task &t) |
static bool | is_proxy (const task &t) |
True if t is a task_proxy. More... | |
static generic_scheduler * | create_master (arena *a) |
Initialize a scheduler for a master thread. More... | |
static generic_scheduler * | create_worker (market &m, size_t index) |
Initialize a scheduler for a worker thread. More... | |
static void | cleanup_worker (void *arg, bool worker) |
Perform necessary cleanup when a worker thread finishes. More... | |
static task * | plugged_return_list () |
Special value used to mark my_return_list as not taking any more entries. More... | |
Public Attributes | |
uintptr_t | my_stealing_threshold |
Position in the call stack specifying its maximal filling when stealing is still allowed. More... | |
market * | my_market |
The market I am in. More... | |
FastRandom | my_random |
Random number generator used for picking a random victim from which to steal. More... | |
task * | my_free_list |
Free list of small tasks that can be reused. More... | |
task * | my_dummy_task |
Fake root task created by slave threads. More... | |
long | my_ref_count |
Reference count for scheduler. More... | |
bool | my_auto_initialized |
True if *this was created by automatic TBB initialization. More... | |
__TBB_atomic intptr_t | my_small_task_count |
Number of small tasks that have been allocated by this scheduler. More... | |
task * | my_return_list |
List of small tasks that have been returned to this scheduler by other schedulers. More... | |
![]() | |
intrusive_list_node * | my_prev_node |
intrusive_list_node * | my_next_node |
![]() | |
size_t | my_arena_index |
Index of the arena slot the scheduler occupies now, or occupied last time. More... | |
arena_slot * | my_arena_slot |
Pointer to the slot in the arena we own at the moment. More... | |
arena * | my_arena |
The arena that I own (if master) or am servicing at the moment (if worker) More... | |
task * | my_innermost_running_task |
Innermost task whose task::execute() is running. A dummy task on the outermost level. More... | |
mail_inbox | my_inbox |
affinity_id | my_affinity_id |
The mailbox id assigned to this scheduler. More... | |
scheduler_properties | my_properties |
Static Public Attributes | |
static const size_t | quick_task_size = 256-task_prefix_reservation_size |
If sizeof(task) is <=quick_task_size, it is handled on a free list instead of malloc'd. More... | |
static const size_t | null_arena_index = ~size_t(0) |
static const size_t | min_task_pool_size = 64 |
Protected Member Functions | |
generic_scheduler (market &) | |
Friends | |
template<typename SchedulerTraits > | |
class | custom_scheduler |
Work stealing task scheduler.
None of the fields here are ever read or written by threads other than the thread that creates the instance.
Class generic_scheduler is an abstract base class that contains most of the scheduler, except for tweaks specific to processors and tools (e.g. VTune). The derived template class custom_scheduler<SchedulerTraits> fills in the tweaks.
Definition at line 124 of file scheduler.h.
|
protected |
Definition at line 88 of file scheduler.cpp.
References __TBB_ASSERT, __TBB_CONTEXT_ARG, allocate_task(), tbb::internal::governor::is_set(), ITT_SYNC_CREATE, tbb::internal::scheduler_state::my_arena_index, my_dummy_task, tbb::internal::scheduler_state::my_innermost_running_task, tbb::internal::scheduler_state::my_properties, my_return_list, tbb::internal::scheduler_properties::outermost, and tbb::task::prefix().
|
inline |
Locks the local task pool.
Garbles my_arena_slot->task_pool for the duration of the lock. Requires correctly set my_arena_slot->task_pool_ptr.
ATTENTION: This method is mostly the same as generic_scheduler::lock_task_pool(), with a little different logic of slot state checks (slot is either locked or points to our task pool). Thus if either of them is changed, consider changing the counterpart as well.
Definition at line 460 of file scheduler.cpp.
References __TBB_ASSERT, tbb::internal::as_atomic(), is_task_pool_published(), ITT_NOTIFY, LockedTaskPool, tbb::internal::scheduler_state::my_arena, tbb::internal::scheduler_state::my_arena_index, tbb::internal::scheduler_state::my_arena_slot, tbb::internal::arena::my_slots, tbb::internal::atomic_backoff::pause(), tbb::internal::arena_slot_line1::task_pool, and tbb::internal::arena_slot_line2::task_pool_ptr.
Referenced by cleanup_master(), get_task(), and prepare_task_pool().
task & tbb::internal::generic_scheduler::allocate_task | ( | size_t | number_of_bytes, |
__TBB_CONTEXT_ARG(task *parent, task_group_context *context) | |||
) |
Allocate task object, either from the heap or a free list.
Returns uninitialized task object with initialized prefix.
Definition at line 304 of file scheduler.cpp.
References __TBB_ASSERT, __TBB_cl_prefetch, __TBB_ISOLATION_EXPR, tbb::task::allocated, tbb::task::freed, GATHER_STATISTIC, ITT_NOTIFY, my_free_list, my_return_list, my_small_task_count, tbb::internal::NFS_Allocate(), tbb::internal::no_isolation, p, parent, tbb::task::prefix(), quick_task_size, tbb::task::state(), and tbb::internal::task_prefix_reservation_size.
Referenced by generic_scheduler(), and prepare_for_spawning().
|
inline |
Definition at line 323 of file scheduler.h.
Referenced by local_spawn(), prepare_task_pool(), and tbb::task::self().
Definition at line 40 of file arena.cpp.
References __TBB_ASSERT, attach_mailbox(), tbb::internal::mail_inbox::is_idle_state(), tbb::internal::scheduler_state::my_arena, tbb::internal::scheduler_state::my_arena_index, tbb::internal::scheduler_state::my_arena_slot, my_dummy_task, tbb::internal::scheduler_state::my_inbox, tbb::internal::arena_base::my_market, my_market, tbb::internal::arena::my_slots, tbb::task::prefix(), and tbb::internal::mail_inbox::set_is_idle().
Referenced by nested_arena_entry().
|
inline |
Definition at line 589 of file scheduler.h.
References __TBB_ASSERT, tbb::internal::mail_inbox::attach(), id, tbb::internal::arena::mailbox(), tbb::internal::scheduler_state::my_affinity_id, tbb::internal::scheduler_state::my_arena, and tbb::internal::scheduler_state::my_inbox.
Referenced by attach_arena().
|
inline |
Returns true if stealing is allowed.
Definition at line 195 of file scheduler.h.
References __TBB_get_bsp(), and my_stealing_threshold.
bool tbb::internal::generic_scheduler::cleanup_master | ( | bool | blocking_terminate | ) |
Perform necessary cleanup when a master thread stops using TBB.
Definition at line 1304 of file scheduler.cpp.
References __TBB_ASSERT, tbb::internal::__TBB_load_relaxed(), tbb::internal::__TBB_store_with_release(), acquire_task_pool(), EmptyTaskPool, free_scheduler(), tbb::internal::arena_slot_line1::head, tbb::internal::governor::is_set(), is_task_pool_published(), leave_task_pool(), local_wait_for_all(), lock, tbb::internal::scheduler_state::my_arena, tbb::internal::scheduler_state::my_arena_slot, my_dummy_task, my_market, tbb::internal::arena_slot_line1::my_scheduler, tbb::internal::arena::my_slots, tbb::internal::NFS_Free(), tbb::internal::arena::on_thread_leaving(), tbb::internal::arena::ref_external, tbb::internal::market::release(), release_task_pool(), tbb::internal::arena_slot_line2::tail, and tbb::internal::arena_slot_line1::task_pool.
Perform necessary cleanup when a worker thread finishes.
Definition at line 1294 of file scheduler.cpp.
References __TBB_ASSERT, and s.
Referenced by tbb::internal::market::cleanup().
|
inline |
Makes relocated tasks visible to thieves and releases the local task pool.
Obviously, the task pool must be locked when calling this method.
Definition at line 641 of file scheduler.h.
References __TBB_ASSERT, tbb::internal::__TBB_store_relaxed(), __TBB_store_release, tbb::internal::arena_slot_line1::head, is_local_task_pool_quiescent(), tbb::internal::scheduler_state::my_arena_slot, release_task_pool(), and tbb::internal::arena_slot_line2::tail.
Referenced by prepare_task_pool().
|
inline |
Makes newly spawned tasks visible to thieves.
Definition at line 632 of file scheduler.h.
References __TBB_ASSERT, tbb::internal::__TBB_store_with_release(), ITT_NOTIFY, tbb::internal::scheduler_state::my_arena_slot, sync_releasing, and tbb::internal::arena_slot_line2::tail.
Referenced by local_spawn().
|
static |
Initialize a scheduler for a master thread.
Definition at line 1252 of file scheduler.cpp.
References __TBB_ASSERT, tbb::internal::allocate_scheduler(), tbb::task_group_context::default_traits, tbb::internal::market::global_market(), tbb::task_group_context::isolated, lock, tbb::internal::scheduler_properties::master, tbb::internal::NFS_Allocate(), tbb::task::prefix(), s, and tbb::internal::governor::sign_on().
Referenced by tbb::internal::governor::init_scheduler(), and tbb::internal::governor::init_scheduler_weak().
|
static |
Initialize a scheduler for a worker thread.
Definition at line 1239 of file scheduler.cpp.
References __TBB_ASSERT, tbb::internal::allocate_scheduler(), s, tbb::internal::governor::sign_on(), and tbb::internal::scheduler_properties::worker.
Referenced by tbb::internal::market::create_one_job().
Return task object to the memory allocator.
Definition at line 605 of file scheduler.h.
References tbb::internal::NFS_Free(), p, tbb::internal::poison_pointer(), tbb::task::prefix(), and tbb::internal::task_prefix_reservation_size.
Referenced by free_nonlocal_small_task(), free_scheduler(), and free_task().
For internal use only.
Implements tbb::internal::scheduler.
Definition at line 714 of file scheduler.cpp.
References __TBB_ASSERT, tbb::internal::governor::local_scheduler(), and s.
Free a small task t that that was allocated by a different scheduler.
Definition at line 379 of file scheduler.cpp.
References __TBB_ASSERT, __TBB_cl_evict, __TBB_FetchAndDecrementWrelease, tbb::internal::as_atomic(), deallocate_task(), tbb::task::freed, ITT_NOTIFY, tbb::internal::NFS_Free(), plugged_return_list(), tbb::task::prefix(), s, tbb::task::state(), and sync_releasing.
Referenced by free_scheduler(), and free_task().
void tbb::internal::generic_scheduler::free_scheduler | ( | ) |
Destroy and deallocate this scheduler object.
Definition at line 264 of file scheduler.cpp.
References __TBB_ASSERT, deallocate_task(), free_nonlocal_small_task(), tbb::internal::scheduler_state::my_arena_slot, my_dummy_task, my_free_list, my_market, tbb::internal::scheduler_state::my_properties, my_return_list, my_small_task_count, tbb::internal::NFS_Free(), p, plugged_return_list(), tbb::task::prefix(), and tbb::internal::governor::sign_off().
Referenced by cleanup_master().
Put task on free list.
Does not call destructor.
Definition at line 652 of file scheduler.h.
References __TBB_ASSERT, tbb::task::allocated, deallocate_task(), tbb::task::executing, free_nonlocal_small_task(), tbb::task::freed, GATHER_STATISTIC, h, tbb::internal::local_task, my_free_list, tbb::internal::no_cache, p, tbb::internal::poison_pointer(), poison_value, tbb::task::prefix(), tbb::internal::small_local_task, tbb::internal::small_task, and tbb::task::state().
Referenced by tbb::interface5::internal::task_base::destroy(), tbb::internal::allocate_additional_child_of_proxy::free(), tbb::internal::allocate_continuation_proxy::free(), tbb::internal::allocate_child_proxy::free(), and tbb::internal::auto_empty_task::~auto_empty_task().
task * tbb::internal::generic_scheduler::get_mailbox_task | ( | __TBB_ISOLATION_EXPR(isolation_tag isolation) | ) |
Attempt to get a task from the mailbox.
Gets a task only if it has not been executed by its sender or a thief that has stolen it from the sender's task pool. Otherwise returns NULL.
This method is intended to be used only by the thread extracting the proxy from its mailbox. (In contrast to local task pool, mailbox can be read only by its owner).
Definition at line 1200 of file scheduler.cpp.
References __TBB_ASSERT, __TBB_ISOLATION_EXPR, tbb::internal::es_task_is_stolen, ITT_NOTIFY, tbb::internal::task_proxy::mailbox_bit, tbb::internal::scheduler_state::my_affinity_id, tbb::internal::scheduler_state::my_inbox, and tbb::internal::mail_inbox::pop().
|
inline |
Get a task from the local pool.
Called only by the pool owner. Returns the pointer to the task or NULL if a suitable task is not found. Resets the pool if it is empty.
Definition at line 978 of file scheduler.cpp.
References __TBB_ASSERT, __TBB_control_consistency_helper, __TBB_ISOLATION_EXPR, tbb::internal::__TBB_load_relaxed(), tbb::internal::__TBB_store_relaxed(), tbb::internal::__TBB_store_with_release(), acquire_task_pool(), tbb::internal::arena::advertise_new_work(), tbb::internal::assert_task_valid(), tbb::atomic_fence(), tbb::internal::arena_slot_line1::head, is_quiescent_local_task_pool_reset(), is_task_pool_published(), tbb::internal::scheduler_state::my_affinity_id, tbb::internal::scheduler_state::my_arena, tbb::internal::scheduler_state::my_arena_slot, tbb::internal::scheduler_state::my_innermost_running_task, tbb::task::note_affinity(), tbb::internal::poison_pointer(), publish_task_pool(), release_task_pool(), reset_task_pool_and_leave(), tbb::internal::arena_slot_line2::tail, tbb::internal::arena_slot_line2::task_pool_ptr, and tbb::internal::arena::wakeup.
|
inline |
Get a task from the local pool at specified location T.
Returns the pointer to the task or NULL if the task cannot be executed, e.g. proxy has been deallocated or isolation constraint is not met. tasks_omitted tells if some tasks have been omitted. Called only by the pool owner. The caller should guarantee that the position T is not available for a thief.
Definition at line 926 of file scheduler.cpp.
References __TBB_ASSERT, tbb::internal::__TBB_load_relaxed(), tbb::internal::task_proxy::extract_task(), GATHER_STATISTIC, tbb::internal::no_isolation, tbb::internal::poison_pointer(), tbb::internal::task_proxy::pool_bit, and tbb::task::prefix().
void tbb::internal::generic_scheduler::init_stack_info | ( | ) |
Sets up the data necessary for the stealing limiting heuristics.
Definition at line 150 of file scheduler.cpp.
References __TBB_ASSERT, __TBB_get_bsp(), is_worker(), tbb::internal::MByte, my_market, my_stealing_threshold, and tbb::internal::market::worker_stack_size().
|
inline |
Definition at line 555 of file scheduler.h.
References __TBB_ASSERT, EmptyTaskPool, LockedTaskPool, tbb::internal::scheduler_state::my_arena_slot, and tbb::internal::arena_slot_line1::task_pool.
Referenced by commit_relocated_tasks(), is_quiescent_local_task_pool_empty(), and is_quiescent_local_task_pool_reset().
|
inlinestatic |
True if t is a task_proxy.
Definition at line 273 of file scheduler.h.
References tbb::internal::es_task_proxy, and tbb::task::prefix().
Referenced by steal_task(), and steal_task_from().
|
inline |
Definition at line 561 of file scheduler.h.
References __TBB_ASSERT, tbb::internal::__TBB_load_relaxed(), tbb::internal::arena_slot_line1::head, is_local_task_pool_quiescent(), tbb::internal::scheduler_state::my_arena_slot, and tbb::internal::arena_slot_line2::tail.
Referenced by leave_task_pool().
|
inline |
Definition at line 566 of file scheduler.h.
References __TBB_ASSERT, tbb::internal::__TBB_load_relaxed(), tbb::internal::arena_slot_line1::head, is_local_task_pool_quiescent(), tbb::internal::scheduler_state::my_arena_slot, and tbb::internal::arena_slot_line2::tail.
Referenced by get_task(), and prepare_task_pool().
|
inline |
Definition at line 550 of file scheduler.h.
References __TBB_ASSERT, EmptyTaskPool, tbb::internal::scheduler_state::my_arena_slot, and tbb::internal::arena_slot_line1::task_pool.
Referenced by acquire_task_pool(), cleanup_master(), get_task(), leave_task_pool(), local_spawn(), prepare_task_pool(), and release_task_pool().
|
inlinestatic |
Definition at line 133 of file scheduler.h.
References tbb::task::prefix().
Referenced by prepare_for_spawning(), and steal_task().
|
inline |
True if running on a worker thread, false otherwise.
Definition at line 595 of file scheduler.h.
References tbb::internal::scheduler_state::my_properties, tbb::internal::scheduler_properties::type, and tbb::internal::scheduler_properties::worker.
Referenced by tbb::internal::market::cleanup(), init_stack_info(), master_outermost_level(), nested_arena_entry(), nested_arena_exit(), and worker_outermost_level().
|
inline |
Leave the task pool.
Leaving task pool automatically releases the task pool if it is locked.
Definition at line 1226 of file scheduler.cpp.
References __TBB_ASSERT, tbb::internal::__TBB_store_relaxed(), EmptyTaskPool, is_quiescent_local_task_pool_empty(), is_task_pool_published(), ITT_NOTIFY, LockedTaskPool, tbb::internal::scheduler_state::my_arena, tbb::internal::scheduler_state::my_arena_index, tbb::internal::scheduler_state::my_arena_slot, tbb::internal::arena::my_slots, sync_releasing, and tbb::internal::arena_slot_line1::task_pool.
Referenced by cleanup_master(), and reset_task_pool_and_leave().
Conceptually, this method should be a member of class scheduler. But doing so would force us to publish class scheduler in the headers.
Definition at line 618 of file scheduler.cpp.
References __TBB_ASSERT, tbb::internal::arena::advertise_new_work(), assert_task_pool_valid(), commit_spawned_tasks(), tbb::internal::fast_reverse_vector< T, max_segments >::copy_memory(), end, tbb::internal::first(), tbb::internal::governor::is_set(), is_task_pool_published(), min_task_pool_size, tbb::internal::scheduler_state::my_arena, tbb::internal::scheduler_state::my_arena_slot, tbb::task::prefix(), prepare_for_spawning(), prepare_task_pool(), publish_task_pool(), tbb::internal::fast_reverse_vector< T, max_segments >::push_back(), tbb::internal::fast_reverse_vector< T, max_segments >::size(), tbb::internal::arena_slot_line2::task_pool_ptr, and tbb::internal::arena::work_spawned.
Referenced by local_spawn_root_and_wait(), spawn(), and tbb::internal::custom_scheduler< SchedulerTraits >::tally_completion_of_predecessor().
Definition at line 685 of file scheduler.cpp.
References __TBB_ASSERT, __TBB_CONTEXT_ARG, tbb::internal::first(), tbb::internal::governor::is_set(), local_spawn(), local_wait_for_all(), tbb::internal::auto_empty_task::prefix(), tbb::task::prefix(), and tbb::internal::task_prefix::ref_count.
Referenced by spawn_root_and_wait().
|
pure virtual |
Implemented in tbb::internal::custom_scheduler< SchedulerTraits >.
Referenced by cleanup_master(), local_spawn_root_and_wait(), and wait_until_empty().
|
inline |
Locks victim's task pool, and returns pointer to it. The pointer can be NULL.
Garbles victim_arena_slot->task_pool for the duration of the lock.
ATTENTION: This method is mostly the same as generic_scheduler::acquire_task_pool(), with a little different logic of slot state checks (slot can be empty, locked or point to any task pool other than ours, and asynchronous transitions between all these states are possible). Thus if any of them is changed, consider changing the counterpart as well
Definition at line 504 of file scheduler.cpp.
References __TBB_ASSERT, __TBB_Yield, tbb::internal::as_atomic(), EmptyTaskPool, GATHER_STATISTIC, ITT_NOTIFY, LockedTaskPool, tbb::internal::scheduler_state::my_arena, tbb::internal::arena_base::my_limit, sync_cancel, and tbb::internal::arena_slot_line1::task_pool.
Referenced by steal_task_from().
|
inline |
True if the scheduler is on the outermost dispatch level in a master thread.
Returns true when this scheduler instance is associated with an application thread, and is not executing any TBB task. This includes being in a TBB dispatch loop (one of wait_for_all methods) invoked directly from that thread.
Definition at line 575 of file scheduler.h.
References is_worker(), and outermost_level().
|
inline |
Returns the concurrency limit of the current arena.
Definition at line 599 of file scheduler.h.
References __TBB_ASSERT, tbb::internal::scheduler_state::my_arena, and tbb::internal::arena_base::my_num_slots.
Referenced by tbb::internal::get_initial_auto_partitioner_divisor(), and tbb::internal::affinity_partitioner_base_v3::resize().
Definition at line 679 of file arena.cpp.
References __TBB_ASSERT, tbb::internal::market::adjust_demand(), tbb::internal::governor::assume_scheduler(), attach_arena(), is_worker(), tbb::internal::scheduler_state::my_arena, tbb::internal::arena_base::my_market, and tbb::internal::arena_base::my_num_reserved_slots.
void tbb::internal::generic_scheduler::nested_arena_exit | ( | ) |
Definition at line 703 of file arena.cpp.
References __TBB_ASSERT, tbb::internal::__TBB_store_with_release(), tbb::internal::market::adjust_demand(), is_worker(), tbb::internal::scheduler_state::my_arena, tbb::internal::scheduler_state::my_arena_index, tbb::internal::arena_base::my_exit_monitors, tbb::internal::arena_base::my_market, tbb::internal::arena_base::my_num_reserved_slots, tbb::internal::arena_slot_line1::my_scheduler, tbb::internal::arena::my_slots, and tbb::internal::concurrent_monitor::notify_one().
Referenced by tbb::internal::nested_arena_context::~nested_arena_context().
|
inline |
True if the scheduler is on the outermost dispatch level.
Definition at line 571 of file scheduler.h.
References tbb::internal::scheduler_state::my_properties, and tbb::internal::scheduler_properties::outermost.
Referenced by master_outermost_level(), and worker_outermost_level().
|
inlinestatic |
Special value used to mark my_return_list as not taking any more entries.
Definition at line 380 of file scheduler.h.
Referenced by free_nonlocal_small_task(), and free_scheduler().
Checks if t is affinitized to another thread, and if so, bundles it as proxy.
Returns either t or proxy containing t.
Definition at line 562 of file scheduler.cpp.
References __TBB_ASSERT, __TBB_CONTEXT_ARG, __TBB_ISOLATION_EXPR, allocate_task(), tbb::task::allocated, tbb::task::context(), tbb::internal::es_ref_count_active, tbb::internal::es_task_proxy, is_version_3_task(), ITT_NOTIFY, tbb::internal::task_proxy::location_mask, tbb::internal::arena::mailbox(), tbb::internal::scheduler_state::my_affinity_id, tbb::internal::scheduler_state::my_arena, tbb::internal::scheduler_state::my_innermost_running_task, tbb::internal::task_proxy::outbox, parent, tbb::task::parent(), tbb::internal::poison_pointer(), tbb::task::prefix(), tbb::internal::mail_outbox::push(), tbb::task::ready, tbb::task::state(), sync_releasing, and tbb::internal::task_proxy::task_and_tag.
Referenced by local_spawn().
|
inline |
Makes sure that the task pool can accommodate at least n more elements.
If necessary relocates existing task pointers or grows the ready task deque. Returns (possible updated) tail index (not accounting for n).
Definition at line 406 of file scheduler.cpp.
References __TBB_ASSERT, tbb::internal::__TBB_load_relaxed(), acquire_task_pool(), tbb::internal::arena_slot::allocate_task_pool(), assert_task_pool_valid(), commit_relocated_tasks(), tbb::internal::arena_slot::fill_with_canary_pattern(), tbb::internal::arena_slot_line1::head, is_quiescent_local_task_pool_reset(), is_task_pool_published(), min_task_pool_size, tbb::internal::scheduler_state::my_arena_slot, tbb::internal::arena_slot_line2::my_task_pool_size, new_size, tbb::internal::NFS_Free(), tbb::internal::arena_slot_line2::tail, and tbb::internal::arena_slot_line2::task_pool_ptr.
Referenced by local_spawn().
|
inline |
Used by workers to enter the task pool.
Does not lock the task pool in case if arena slot has been successfully grabbed.
Definition at line 1214 of file scheduler.cpp.
References __TBB_ASSERT, tbb::internal::__TBB_load_relaxed(), tbb::internal::__TBB_store_with_release(), EmptyTaskPool, tbb::internal::arena_slot_line1::head, ITT_NOTIFY, tbb::internal::scheduler_state::my_arena, tbb::internal::scheduler_state::my_arena_index, tbb::internal::scheduler_state::my_arena_slot, tbb::internal::arena::my_slots, sync_releasing, tbb::internal::arena_slot_line2::tail, tbb::internal::arena_slot_line1::task_pool, and tbb::internal::arena_slot_line2::task_pool_ptr.
Referenced by get_task(), and local_spawn().
|
pure virtual |
Try getting a task from other threads (via mailbox, stealing, FIFO queue, orphans adoption).
Returns obtained task or NULL if all attempts fail.
Implemented in tbb::internal::custom_scheduler< SchedulerTraits >.
|
inline |
Unlocks the local task pool.
Restores my_arena_slot->task_pool munged by acquire_task_pool. Requires correctly set my_arena_slot->task_pool_ptr.
Definition at line 489 of file scheduler.cpp.
References __TBB_ASSERT, tbb::internal::__TBB_store_with_release(), is_task_pool_published(), ITT_NOTIFY, LockedTaskPool, tbb::internal::scheduler_state::my_arena_slot, sync_releasing, tbb::internal::arena_slot_line1::task_pool, and tbb::internal::arena_slot_line2::task_pool_ptr.
Referenced by cleanup_master(), commit_relocated_tasks(), and get_task().
|
inline |
Resets head and tail indices to 0, and leaves task pool.
The task pool must be locked by the owner (via acquire_task_pool).
Definition at line 624 of file scheduler.h.
References __TBB_ASSERT, tbb::internal::__TBB_store_relaxed(), tbb::internal::arena_slot_line1::head, leave_task_pool(), LockedTaskPool, tbb::internal::scheduler_state::my_arena_slot, tbb::internal::arena_slot_line2::tail, and tbb::internal::arena_slot_line1::task_pool.
Referenced by get_task().
For internal use only.
Implements tbb::internal::scheduler.
Definition at line 706 of file scheduler.cpp.
References tbb::internal::first(), tbb::internal::governor::local_scheduler(), and local_spawn().
For internal use only.
Implements tbb::internal::scheduler.
Definition at line 710 of file scheduler.cpp.
References tbb::internal::first(), tbb::internal::governor::local_scheduler(), and local_spawn_root_and_wait().
task * tbb::internal::generic_scheduler::steal_task | ( | __TBB_ISOLATION_EXPR(isolation_tag isolation) | ) |
Attempts to steal a task from a randomly chosen thread/scheduler.
Definition at line 1075 of file scheduler.cpp.
References __TBB_ISOLATION_ARG, EmptyTaskPool, tbb::internal::es_task_is_stolen, tbb::internal::task_proxy::extract_task(), GATHER_STATISTIC, tbb::internal::FastRandom::get(), is_proxy(), is_version_3_task(), tbb::internal::scheduler_state::my_affinity_id, tbb::internal::scheduler_state::my_arena, tbb::internal::scheduler_state::my_arena_index, tbb::internal::scheduler_state::my_innermost_running_task, tbb::internal::arena_base::my_limit, my_random, tbb::internal::arena::my_slots, tbb::task::note_affinity(), tbb::internal::task_proxy::pool_bit, tbb::task::prefix(), steal_task_from(), and tbb::internal::arena_slot_line1::task_pool.
task * tbb::internal::generic_scheduler::steal_task_from | ( | __TBB_ISOLATION_ARG(arena_slot &victim_arena_slot, isolation_tag isolation) | ) |
Steal task from another scheduler's ready pool.
Definition at line 1110 of file scheduler.cpp.
References __TBB_ASSERT, __TBB_cl_evict, __TBB_control_consistency_helper, __TBB_ISOLATION_EXPR, tbb::internal::__TBB_load_relaxed(), tbb::internal::__TBB_store_relaxed(), tbb::internal::arena::advertise_new_work(), tbb::atomic_fence(), GATHER_STATISTIC, tbb::internal::arena_slot_line1::head, is_proxy(), tbb::internal::task_proxy::is_shared(), ITT_NOTIFY, lock_task_pool(), tbb::internal::scheduler_state::my_arena, tbb::internal::no_isolation, tbb::internal::task_proxy::outbox, tbb::internal::poison_pointer(), tbb::task::prefix(), tbb::internal::mail_outbox::recipient_is_idle(), tbb::internal::arena_slot_line2::tail, tbb::internal::task_proxy::task_and_tag, unlock_task_pool(), and tbb::internal::arena::wakeup.
Referenced by steal_task().
|
inline |
Unlocks victim's task pool.
Restores victim_arena_slot->task_pool munged by lock_task_pool.
Definition at line 553 of file scheduler.cpp.
References __TBB_ASSERT, tbb::internal::__TBB_store_with_release(), ITT_NOTIFY, LockedTaskPool, sync_releasing, and tbb::internal::arena_slot_line1::task_pool.
Referenced by steal_task_from().
void tbb::internal::generic_scheduler::wait_until_empty | ( | ) |
Definition at line 719 of file arena.cpp.
References local_wait_for_all(), tbb::internal::scheduler_state::my_arena, my_dummy_task, tbb::internal::arena_base::my_pool_state, tbb::task::prefix(), and tbb::internal::arena::SNAPSHOT_EMPTY.
|
inline |
True if the scheduler is on the outermost dispatch level in a worker thread.
Definition at line 579 of file scheduler.h.
References is_worker(), and outermost_level().
|
friend |
Definition at line 314 of file scheduler.h.
|
static |
Initial size of the task deque sufficient to serve without reallocation 4 nested parallel_for calls with iteration space of 65535 grains each.
Definition at line 294 of file scheduler.h.
Referenced by local_spawn(), and prepare_task_pool().
bool tbb::internal::generic_scheduler::my_auto_initialized |
True if *this was created by automatic TBB initialization.
Definition at line 184 of file scheduler.h.
task* tbb::internal::generic_scheduler::my_dummy_task |
Fake root task created by slave threads.
The task is used as the "parent" argument to method wait_for_all.
Definition at line 173 of file scheduler.h.
Referenced by attach_arena(), cleanup_master(), free_scheduler(), generic_scheduler(), tbb::internal::nested_arena_context::mimic_outermost_level(), wait_until_empty(), and tbb::internal::nested_arena_context::~nested_arena_context().
task* tbb::internal::generic_scheduler::my_free_list |
Free list of small tasks that can be reused.
Definition at line 165 of file scheduler.h.
Referenced by allocate_task(), free_scheduler(), and free_task().
market* tbb::internal::generic_scheduler::my_market |
The market I am in.
Definition at line 159 of file scheduler.h.
Referenced by attach_arena(), cleanup_master(), free_scheduler(), and init_stack_info().
FastRandom tbb::internal::generic_scheduler::my_random |
Random number generator used for picking a random victim from which to steal.
Definition at line 162 of file scheduler.h.
Referenced by steal_task(), and tbb::internal::custom_scheduler< SchedulerTraits >::tally_completion_of_predecessor().
long tbb::internal::generic_scheduler::my_ref_count |
Reference count for scheduler.
Number of task_scheduler_init objects that point to this scheduler
Definition at line 177 of file scheduler.h.
task* tbb::internal::generic_scheduler::my_return_list |
List of small tasks that have been returned to this scheduler by other schedulers.
Definition at line 387 of file scheduler.h.
Referenced by allocate_task(), free_scheduler(), and generic_scheduler().
__TBB_atomic intptr_t tbb::internal::generic_scheduler::my_small_task_count |
Number of small tasks that have been allocated by this scheduler.
Definition at line 383 of file scheduler.h.
Referenced by allocate_task(), and free_scheduler().
uintptr_t tbb::internal::generic_scheduler::my_stealing_threshold |
Position in the call stack specifying its maximal filling when stealing is still allowed.
Definition at line 142 of file scheduler.h.
Referenced by can_steal(), and init_stack_info().
|
static |
Definition at line 148 of file scheduler.h.
|
static |
If sizeof(task) is <=quick_task_size, it is handled on a free list instead of malloc'd.
Definition at line 131 of file scheduler.h.
Referenced by allocate_task().