32 #if __TBB_STATISTICS_STDOUT 51 #if __TBB_TASK_GROUP_CONTEXT 56 #if __TBB_TASK_PRIORITY 63 my_ref_top_priority = &a->my_top_priority;
64 my_ref_reload_epoch = &a->my_reload_epoch;
66 my_local_reload_epoch = *my_ref_reload_epoch;
72 return !slot &&
as_atomic( slot ).compare_and_swap( &
s, NULL ) == NULL;
78 size_t index =
s.my_arena_index;
79 if ( index < lower || index >= upper ) index =
s.my_random.get() % (upper - lower) + lower;
82 for (
size_t i = index; i < upper; ++i )
84 for (
size_t i = lower; i < index; ++i )
89 template <
bool as_worker>
109 __TBB_ASSERT(
s.my_innermost_running_task ==
s.my_dummy_task, NULL );
119 s.attach_arena(
this, index,
false );
121 #if !__TBB_FP_CONTEXT 125 #if __TBB_ARENA_OBSERVER 126 __TBB_ASSERT( !
s.my_last_local_observer,
"There cannot be notified local observers when entering arena" );
127 my_observers.notify_entry_observers(
s.my_last_local_observer,
true );
133 s.local_wait_for_all( *
s.my_dummy_task, NULL );
138 __TBB_ASSERT(
s.my_innermost_running_task ==
s.my_dummy_task, NULL );
142 "Worker cannot leave arena while its task pool is not reset" );
148 || recall_by_mandatory_request()
159 s.my_innermost_running_task =
s.my_dummy_task;
160 s.local_wait_for_all(*
s.my_dummy_task,t);
163 #if __TBB_ARENA_OBSERVER 164 my_observers.notify_exit_observers(
s.my_last_local_observer,
true );
165 s.my_last_local_observer = NULL;
167 #if __TBB_TASK_PRIORITY 168 if (
s.my_offloaded_tasks )
169 orphan_offloaded_tasks(
s );
172 ++
s.my_counters.arena_roundtrips;
173 *
my_slots[index].my_counters +=
s.my_counters;
174 s.my_counters.reset();
180 __TBB_ASSERT(
s.my_innermost_running_task ==
s.my_dummy_task, NULL );
187 on_thread_leaving<ref_worker>();
191 __TBB_ASSERT( !my_guard,
"improperly allocated arena?" );
194 #if __TBB_TASK_PRIORITY 195 __TBB_ASSERT( !my_reload_epoch && !my_orphaned_tasks && !my_skipped_fifo_priority,
"New arena object is not zeroed" );
204 #if __TBB_TASK_PRIORITY 205 my_bottom_priority = my_top_priority = normalized_normal_priority;
208 #if __TBB_ARENA_OBSERVER 209 my_observers.my_arena =
this;
221 #if __TBB_PREVIEW_CRITICAL_TASKS 225 my_slots[i].my_counters =
new (
NFS_Allocate(1,
sizeof(statistics_counters), NULL) ) statistics_counters;
230 #if __TBB_PREVIEW_CRITICAL_TASKS 232 ITT_SYNC_CREATE(&my_critical_task_stream, SyncType_Scheduler, SyncObj_CriticalTaskStream);
234 #if __TBB_ENQUEUE_ENFORCED_CONCURRENCY 235 my_concurrency_mode = cm_normal;
237 #if !__TBB_FP_CONTEXT 247 unsigned char* storage = (
unsigned char*)
NFS_Allocate( 1, n, NULL );
249 memset( storage, 0, n );
258 #if __TBB_ENQUEUE_ENFORCED_CONCURRENCY 259 __TBB_ASSERT( my_concurrency_mode != cm_enforced_global, NULL );
261 #if !__TBB_STATISTICS_EARLY_DUMP 265 intptr_t drained = 0;
278 #if __TBB_PREVIEW_CRITICAL_TASKS 279 __TBB_ASSERT( my_critical_task_stream.drain()==0,
"Not all critical tasks were executed");
281 #if __TBB_COUNT_TASK_NODES 282 my_market->update_task_node_count( -drained );
286 #if __TBB_TASK_GROUP_CONTEXT 287 __TBB_ASSERT( my_default_ctx,
"Master thread never entered the arena?" );
288 my_default_ctx->~task_group_context();
291 #if __TBB_ARENA_OBSERVER 292 if ( !my_observers.empty() )
293 my_observers.clear();
299 #if TBB_USE_ASSERT > 1 306 void arena::dump_arena_statistics () {
307 statistics_counters total;
309 #if __TBB_STATISTICS_EARLY_DUMP 312 *
my_slots[i].my_counters +=
s->my_counters;
318 dump_statistics( *
my_slots[i].my_counters, i );
321 dump_statistics( *
my_slots[0].my_counters, 0 );
322 #if __TBB_STATISTICS_STDOUT 323 #if !__TBB_STATISTICS_TOTALS_ONLY 324 printf(
"----------------------------------------------\n" );
326 dump_statistics( total, workers_counters_total );
328 dump_statistics( total, arena_counters_total );
329 #if !__TBB_STATISTICS_TOTALS_ONLY 330 printf(
"==============================================\n" );
336 #if __TBB_TASK_PRIORITY 341 inline bool arena::may_have_tasks ( generic_scheduler*
s,
bool& tasks_present,
bool& dequeuing_possible ) {
342 if ( !
s ||
s->my_arena !=
this )
344 dequeuing_possible |=
s->worker_outermost_level();
345 if (
s->my_pool_reshuffling_pending ) {
348 tasks_present =
true;
351 if (
s->my_offloaded_tasks ) {
352 tasks_present =
true;
353 if (
s->my_local_reload_epoch < *
s->my_ref_reload_epoch ) {
362 void arena::orphan_offloaded_tasks(generic_scheduler&
s) {
365 ++my_abandonment_epoch;
366 __TBB_ASSERT(
s.my_offloaded_task_list_tail_link && !*
s.my_offloaded_task_list_tail_link, NULL );
369 orphans = const_cast<task*>(my_orphaned_tasks);
370 *
s.my_offloaded_task_list_tail_link = orphans;
371 }
while (
as_atomic(my_orphaned_tasks).compare_and_swap(
s.my_offloaded_tasks, orphans) != orphans );
372 s.my_offloaded_tasks = NULL;
374 s.my_offloaded_task_list_tail_link = NULL;
394 advertise_new_work<work_enqueued>();
395 #if __TBB_TASK_PRIORITY 400 if ( p < my_bottom_priority || p > my_top_priority )
426 #if __TBB_TASK_PRIORITY 428 intptr_t top_priority = my_top_priority;
432 for( k=0; k<n; ++k ) {
443 bool work_absent = k == n;
444 #if __TBB_PREVIEW_CRITICAL_TASKS 445 bool no_critical_tasks = my_critical_task_stream.empty(0);
446 work_absent &= no_critical_tasks;
448 #if __TBB_TASK_PRIORITY 451 bool tasks_present = !work_absent || my_orphaned_tasks;
452 bool dequeuing_possible =
false;
457 uintptr_t abandonment_epoch = my_abandonment_epoch;
463 the_context_state_propagation_mutex.lock();
464 work_absent = !may_have_tasks(
my_slots[0].my_scheduler, tasks_present, dequeuing_possible );
465 the_context_state_propagation_mutex.unlock();
478 for( k = 1; work_absent && k < n; ++k ) {
481 work_absent = !may_have_tasks(
my_slots[k].my_scheduler, tasks_present, dequeuing_possible );
484 work_absent = work_absent
486 && abandonment_epoch == my_abandonment_epoch;
491 #if __TBB_TASK_PRIORITY 493 work_absent = work_absent && (!dequeuing_possible || no_fifo_tasks)
494 && top_priority == my_top_priority && reload_epoch == my_reload_epoch;
497 work_absent = work_absent && no_fifo_tasks;
500 #if __TBB_TASK_PRIORITY 501 if ( top_priority > my_bottom_priority ) {
502 if (
my_market->lower_arena_priority(*
this, top_priority - 1, reload_epoch)
505 atomic_update( my_skipped_fifo_priority, top_priority, std::less<intptr_t>());
508 else if ( !tasks_present && !my_orphaned_tasks && no_fifo_tasks ) {
514 #if __TBB_ENQUEUE_ENFORCED_CONCURRENCY 515 if( my_concurrency_mode==cm_enforced_global ) {
517 my_market->mandatory_concurrency_disable(
this );
529 #if __TBB_TASK_PRIORITY 546 #if __TBB_COUNT_TASK_NODES 547 intptr_t arena::workers_task_node_count() {
552 result +=
s->my_task_node_count;
560 #if __TBB_RECYCLE_TO_ENQUEUE 571 __TBB_ASSERT( ref_count!=0,
"attempt to enqueue task whose parent has a ref_count==0 (forgot to set_ref_count?)" );
572 __TBB_ASSERT( ref_count>0,
"attempt to enqueue task whose parent has a ref_count<0" );
577 #if __TBB_PREVIEW_CRITICAL_TASKS 581 #if __TBB_TASK_ISOLATION 583 __TBB_ASSERT(
s,
"Scheduler must be initialized at this moment" );
585 t.
prefix().isolation =
s->my_innermost_running_task->prefix().isolation;
588 if( !
s || !
s->my_arena_slot ) {
591 my_critical_task_stream.push( &t, 0, internal::random_lane_selector(random) );
593 unsigned& lane =
s->my_arena_slot->hint_for_critical;
596 advertise_new_work<work_spawned>();
602 #if __TBB_TASK_PRIORITY 603 intptr_t
p = prio ? normalize_priority(
priority_t(prio)) : normalized_normal_priority;
604 assert_priority_valid(
p);
605 #if __TBB_PREVIEW_CRITICAL_TASKS && __TBB_CPF_BUILD 610 if (
p != my_top_priority )
613 __TBB_ASSERT_EX(prio == 0,
"the library is not configured to respect the task priority");
614 #if __TBB_PREVIEW_CRITICAL_TASKS && __TBB_CPF_BUILD 620 advertise_new_work<work_enqueued>();
621 #if __TBB_TASK_PRIORITY 622 if (
p != my_top_priority )
638 s->nested_arena_entry(a, slot_index);
642 #if __TBB_TASK_GROUP_CONTEXT 651 #if __TBB_TASK_PRIORITY 668 #if __TBB_PREVIEW_CRITICAL_TASKS 671 #if __TBB_TASK_GROUP_CONTEXT 684 #if __TBB_TASK_PRIORITY 685 if ( my_offloaded_tasks )
686 my_arena->orphan_offloaded_tasks( *
this );
687 my_offloaded_tasks = NULL;
697 #if __TBB_ARENA_OBSERVER 698 my_last_local_observer = 0;
699 my_arena->my_observers.notify_entry_observers( my_last_local_observer,
false );
704 #if __TBB_ARENA_OBSERVER 705 my_arena->my_observers.notify_exit_observers( my_last_local_observer,
false );
707 #if __TBB_TASK_PRIORITY 708 if ( my_offloaded_tasks )
709 my_arena->orphan_offloaded_tasks( *
this );
733 namespace interface7 {
745 #if __TBB_TASK_GROUP_CONTEXT 749 new_arena->my_default_ctx->capture_fp_settings();
758 #if __TBB_TASK_GROUP_CONTEXT 771 my_arena->my_market->release(
true,
false );
774 #if __TBB_TASK_GROUP_CONTEXT 783 if(
s &&
s->my_arena ) {
789 #if __TBB_TASK_GROUP_CONTEXT 805 #if __TBB_TASK_GROUP_CONTEXT 808 "The task will not be executed because default task_group_context of task_arena is cancelled. Has previously enqueued task thrown an exception?");
810 my_arena->enqueue_task( t, prio,
s->my_random );
819 __TBB_ASSERT(
s.outermost_level(),
"expected to be enqueued and received on the outermost level");
820 struct outermost_context : internal::no_copy {
827 : t(_t),
s(_s), orig_dummy(
s.my_dummy_task), orig_props(
s.my_properties) {
829 #if __TBB_TASK_GROUP_CONTEXT 830 orig_ctx = t->prefix().context;
831 t->prefix().context =
s.my_arena->my_default_ctx;
837 ~outermost_context() {
838 #if __TBB_TASK_GROUP_CONTEXT 840 t->prefix().context = orig_ctx;
842 s.my_properties = orig_props;
843 s.my_dummy_task = orig_dummy;
867 bool same_arena =
s->my_arena ==
my_arena;
868 size_t index1 =
s->my_arena_index;
870 index1 =
my_arena->occupy_free_slot<
false>(*s);
873 #if __TBB_USE_OPTIONAL_RTTI 881 internal::delegated_function< graph_funct, void >* deleg_funct =
882 dynamic_cast< internal::delegated_function< graph_funct, void>*
>(&
d);
887 (internal::forward< graph_funct >(deleg_funct->my_func)), 0);
892 #if __TBB_TASK_GROUP_CONTEXT 905 my_arena->my_exit_monitors.prepare_wait(waiter, (uintptr_t)&
d);
907 my_arena->my_exit_monitors.cancel_wait(waiter);
910 index2 =
my_arena->occupy_free_slot<
false>(*s);
912 my_arena->my_exit_monitors.cancel_wait(waiter);
914 s->local_wait_for_all(root, NULL);
915 #if TBB_USE_EXCEPTIONS 921 my_arena->my_exit_monitors.commit_wait(waiter);
926 my_arena->my_exit_monitors.notify_one();
928 #if TBB_USE_EXCEPTIONS 931 TbbRethrowException(pe);
934 #if __TBB_USE_OPTIONAL_RTTI 942 #if TBB_USE_EXCEPTIONS 948 #if TBB_USE_EXCEPTIONS 971 __TBB_ASSERT(
s->outermost_level(),
"The enqueued task can be processed only on outermost level" );
972 if (
s->is_worker() ) {
975 s->my_innermost_running_task =
s->my_dummy_task;
976 s->local_wait_for_all( *
s->my_dummy_task, NULL );
977 s->my_innermost_running_task =
this;
978 }
else s->my_arena->is_out_of_work();
990 __TBB_ASSERT(
s->my_arena !=
my_arena ||
s->my_arena_index == 0,
"task_arena::wait_until_empty() is not supported within a worker context" );
994 if( !
s->my_arena_index )
995 while(
my_arena->num_workers_active() )
996 s->wait_until_empty();
1000 &&
as_atomic(
my_arena->my_slots[0].my_scheduler).compare_and_swap(
s, NULL) == NULL ) {
1002 s->wait_until_empty();
1009 if( !
my_arena->num_workers_active() && !
my_arena->my_slots[0].my_scheduler)
1017 return s?
int(
s->my_arena_index) : -1;
1020 #if __TBB_TASK_ISOLATION 1025 isolation_guard(
isolation_tag &isolation ) : guarded( isolation ), previous_value( isolation ) {}
1026 ~isolation_guard() {
1027 guarded = previous_value;
1035 __TBB_ASSERT(
s,
"this_task_arena::isolate() needs an initialized scheduler" );
1038 isolation_tag& current_isolation =
s->my_innermost_running_task->prefix().isolation;
1040 isolation_guard guard( current_isolation );
1041 current_isolation = reinterpret_cast<isolation_tag>(&
d);
task * execute() __TBB_override
Should be overridden by derived classes.
intptr_t drain()
Destroys all remaining tasks in every lane. Returns the number of destroyed tasks.
void *__TBB_EXPORTED_FUNC NFS_Allocate(size_t n_element, size_t element_size, void *hint)
Allocate memory on cache/sector line boundary.
void free_arena()
Completes arena shutdown, destructs and deallocates it.
uintptr_t my_arenas_aba_epoch
ABA prevention marker to assign to newly created arenas.
void __TBB_EXPORTED_METHOD register_pending_exception()
Records the pending exception, and cancels the task group.
static int unsigned num_arena_slots(unsigned num_slots)
static int allocation_size(unsigned num_slots)
__TBB_atomic reference_count ref_count
Reference count used for synchronization.
#define __TBB_CONTEXT_ARG(arg1, context)
intptr_t drain()
Drain the mailbox.
bool operator()(uintptr_t ctx) const
unsigned my_num_workers_allotted
The number of workers that have been marked out by the resource manager to service the arena.
nested_arena_context(generic_scheduler *s, arena *a, size_t slot_index, bool type, bool same)
void __TBB_EXPORTED_METHOD internal_initialize()
void set_ctx(__TBB_CONTEXT_ARG1(task_group_context *))
task_group_context * my_context
default context of the arena
void initialize(unsigned n_lanes)
scheduler_state my_orig_state
unsigned short affinity_id
An id as used for specifying affinity.
#define __TBB_ENQUEUE_ENFORCED_CONCURRENCY
size_t __TBB_EXPORTED_FUNC NFS_GetLineSize()
Cache/sector line size.
T __TBB_load_with_acquire(const volatile T &location)
void attach_arena(arena *, size_t index, bool is_master)
market * my_market
The market that owns this arena.
unsigned my_num_reserved_slots
The number of reserved slots (can be occupied only by masters).
Set if ref_count might be changed by another thread. Used for debugging.
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain ITT_FORMAT p const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_id __itt_string_handle __itt_metadata_type size_t void ITT_FORMAT p const __itt_domain __itt_id __itt_string_handle const wchar_t size_t ITT_FORMAT lu const __itt_domain __itt_id head
#define __TBB_ASSERT(predicate, comment)
No-op version of __TBB_ASSERT.
const isolation_tag no_isolation
void adjust_demand(arena &, int delta)
Request that arena's need in workers should be adjusted.
size_t occupy_free_slot(generic_scheduler &s)
Tries to occupy a slot in the arena. On success, returns the slot index; if no slot is available,...
static const size_t out_of_arena
task object is freshly allocated or recycled.
cpu_ctl_env my_cpu_ctl_env
FPU control settings of arena's master thread captured at the moment of arena instantiation.
#define ITT_SYNC_CREATE(obj, type, name)
static market & global_market(bool is_public, unsigned max_num_workers=0, size_t stack_size=0)
Factory method creating new market object.
static const pool_state_t SNAPSHOT_FULL
At least one task has been offered for stealing since the last snapshot started.
void on_thread_leaving()
Notification that worker or master leaves its arena.
Used to form groups of tasks.
A fast random number generator.
Bit-field representing properties of a sheduler.
wait_task(binary_semaphore &sema)
static unsigned default_num_threads()
internal::delegate_base & my_delegate
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d
exception_container_type * my_exception
Pointer to the container storing exception being propagated across this task group.
task * execute() __TBB_override
Should be overridden by derived classes.
void mimic_outermost_level(arena *a, bool type)
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain ITT_FORMAT p const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_id __itt_string_handle __itt_metadata_type type
bool is_worker() const
True if running on a worker thread, false otherwise.
Base class for types that should not be copied or assigned.
intptr_t my_version_and_traits
Special settings.
static generic_scheduler * local_scheduler_if_initialized()
concurrent_monitor & my_monitor
delegated_task(internal::delegate_base &d, concurrent_monitor &s, task *t)
int my_max_concurrency
Concurrency level for deferred initialization.
static arena * create_arena(int num_slots, int num_reserved_slots, size_t stack_size)
Creates an arena object.
void __TBB_EXPORTED_METHOD internal_terminate()
void enqueue_task(task &, intptr_t, FastRandom &)
enqueue a task into starvation-resistance queue
void notify_one()
Notify one thread about the event.
Base class for user-defined tasks.
static bool occupy_slot(generic_scheduler *&slot, generic_scheduler &s)
Work stealing task scheduler.
task * my_innermost_running_task
Innermost task whose task::execute() is running. A dummy task on the outermost level.
static void assume_scheduler(generic_scheduler *s)
Temporarily set TLS slot to the given scheduler.
#define __TBB_ASSERT_EX(predicate, comment)
"Extended" version is useful to suppress warnings if a variable is only used with an assert
static const unsigned ref_external
Reference increment values for externals and workers.
intptr_t isolation_tag
A tag for task isolation.
unsigned my_num_slots
The number of slots in the arena.
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain ITT_FORMAT p const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_id __itt_string_handle __itt_metadata_type size_t void ITT_FORMAT p const __itt_domain __itt_id __itt_string_handle const wchar_t size_t ITT_FORMAT lu const __itt_domain __itt_id __itt_relation __itt_id ITT_FORMAT p const wchar_t int ITT_FORMAT __itt_group_mark d __itt_event ITT_FORMAT __itt_group_mark d void const wchar_t const wchar_t int ITT_FORMAT __itt_group_sync __itt_group_fsync x void const wchar_t int const wchar_t int int ITT_FORMAT __itt_group_sync __itt_group_fsync x void ITT_FORMAT __itt_group_sync __itt_group_fsync p void ITT_FORMAT __itt_group_sync __itt_group_fsync p void size_t ITT_FORMAT lu no args __itt_obj_prop_t __itt_obj_state_t ITT_FORMAT d const char ITT_FORMAT s __itt_frame ITT_FORMAT p const char const char ITT_FORMAT s __itt_counter ITT_FORMAT p __itt_counter unsigned long long ITT_FORMAT lu const wchar_t ITT_FORMAT S __itt_mark_type const wchar_t ITT_FORMAT S __itt_mark_type const char ITT_FORMAT s __itt_mark_type ITT_FORMAT d __itt_caller ITT_FORMAT p __itt_caller ITT_FORMAT p no args const __itt_domain __itt_clock_domain unsigned long long __itt_id ITT_FORMAT lu const __itt_domain __itt_clock_domain unsigned long long __itt_id __itt_id void ITT_FORMAT p const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_clock_domain unsigned long long __itt_id __itt_string_handle __itt_scope scope
arena(market &, unsigned max_num_workers, unsigned num_reserved_slots)
Constructor.
generic_scheduler * my_scheduler
Scheduler of the thread attached to the slot.
market * my_market
The market I am in.
void const char const char int ITT_FORMAT __itt_group_sync p
binary_semaphore for concurrent monitor
size_t my_arena_index
Index of the arena slot the scheduler occupies now, or occupied last time.
bool is_critical(task &t)
void __TBB_EXPORTED_METHOD internal_wait() const
static const int automatic
Typedef for number of threads that is automatic.
Smart holder for the empty task class with automatic destruction.
arena_slot * my_arena_slot
Pointer to the slot in the arena we own at the moment.
#define __TBB_ISOLATION_ARG(arg1, isolation)
task * parent() const
task on whose behalf this task is working, or NULL if this is a root.
size_t occupy_free_slot_in_range(generic_scheduler &s, size_t lower, size_t upper)
Tries to occupy a slot in the specified range.
T1 atomic_update(tbb::atomic< T1 > &dst, T2 newValue, Pred compare)
Atomically replaces value of dst with newValue if they satisfy condition of compare predicate.
tbb::atomic< uintptr_t > my_pool_state
Current task pool state and estimate of available tasks amount.
bool empty(int level)
Checks existence of a task.
atomic< unsigned > my_references
Reference counter for the arena.
bool is_out_of_work()
Check if there is job anywhere in arena.
unsigned my_max_num_workers
The number of workers requested by the master thread owning the arena.
static const pool_state_t SNAPSHOT_EMPTY
No tasks to steal since last snapshot was taken.
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task * task
static const intptr_t num_priority_levels
void construct()
Construct *this as a mailbox from zeroed memory.
int my_num_workers_requested
The number of workers that are currently requested from the resource manager.
void __TBB_EXPORTED_FUNC isolate_within_arena(delegate_base &d, intptr_t reserved=0)
void spin_wait_while_eq(const volatile T &location, U value)
Spin WHILE the value of the variable is equal to a given value.
#define GATHER_STATISTIC(x)
static void one_time_init()
bool release(bool is_public, bool blocking_terminate)
Decrements market's refcount and destroys it in the end.
static arena & allocate_arena(market &, unsigned num_slots, unsigned num_reserved_slots)
Allocate an instance of arena.
bool type
Indicates that a scheduler acts as a master or a worker.
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id parent
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain ITT_FORMAT p const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_id __itt_string_handle __itt_metadata_type size_t void ITT_FORMAT p const __itt_domain __itt_id __itt_string_handle const wchar_t size_t ITT_FORMAT lu const __itt_domain __itt_id __itt_relation __itt_id tail
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p sync_releasing
void push(task *source, int level, FastRandom &random)
Push a task into a lane.
bool has_enqueued_tasks()
Check for the presence of enqueued tasks at all priority levels.
internal::tbb_exception_ptr exception_container_type
task is in ready pool, or is going to be put there, or was just taken off.
void restore_priority_if_need()
If enqueued tasks found, restore arena priority and task presence status.
bool outermost
Indicates that a scheduler is on outermost level.
scheduler_properties my_properties
atomic< unsigned > my_limit
The maximal number of currently busy slots.
uintptr_t my_version_and_traits
Version for run-time checks and behavioral traits of the context.
unsigned my_master_slots
Reserved master slots.
#define ITT_NOTIFY(name, obj)
void attach_mailbox(affinity_id id)
uintptr_t my_aba_epoch
ABA prevention marker.
const size_t NFS_MaxLineSize
Compile-time constant that is upper bound on cache line/sector size.
void const char const char int ITT_FORMAT __itt_group_sync s
#define __TBB_CONTEXT_ARG1(context)
task * my_dummy_task
Fake root task created by slave threads.
static generic_scheduler * local_scheduler_weak()
void notify(const P &predicate)
Notify waiting threads of the event that satisfies the given predicate.
T __TBB_load_relaxed(const volatile T &location)
generic_scheduler & my_scheduler
mail_outbox & mailbox(affinity_id id)
Get reference to mailbox corresponding to given affinity_id.
static bool is_set(generic_scheduler *s)
Used to check validity of the local scheduler TLS contents.
concurrent_monitor my_exit_monitors
Waiting object for master threads that cannot join the arena.
void __TBB_EXPORTED_FUNC NFS_Free(void *)
Free memory allocated by NFS_Allocate.
static int __TBB_EXPORTED_FUNC internal_current_slot()
void copy_fp_settings(const task_group_context &src)
Copies FPU control setting from another context.
void free_task_pool()
Deallocate task pool that was allocated by means of allocate_task_pool.
virtual void local_wait_for_all(task &parent, task *child)=0
value_type compare_and_swap(value_type value, value_type comparand)
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain ITT_FORMAT p const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_id __itt_string_handle __itt_metadata_type size_t void ITT_FORMAT p const __itt_domain __itt_id __itt_string_handle const wchar_t size_t ITT_FORMAT lu const __itt_domain __itt_id __itt_relation __itt_id ITT_FORMAT p const wchar_t int ITT_FORMAT __itt_group_mark d int
task_stream< num_priority_levels > my_task_stream
Task pool for the tasks scheduled via task::enqueue() method.
void set_is_idle(bool value)
Indicate whether thread that reads this mailbox is idle.
internal::arena * my_arena
NULL if not currently initialized.
state_type state() const
Current execution state.
static internal::allocate_root_proxy allocate_root()
Returns proxy for overloaded new that allocates a root task.
void make_critical(task &t)
intptr_t reference_count
A reference count.
static int __TBB_EXPORTED_FUNC internal_max_concurrency(const task_arena *)
static const int priority_critical
internal::task_prefix & prefix(internal::version_tag *=NULL) const
Get reference to corresponding task_prefix.
bool is_idle_state(bool value) const
Indicate whether thread that reads this mailbox is idle.
Class representing where mail is put.
task_group_context * my_orig_ctx
void __TBB_EXPORTED_METHOD internal_execute(delegate_base &) const
binary_semaphore & my_signal
unsigned num_workers_active()
The number of workers active in the arena.
void __TBB_store_with_release(volatile T &location, V value)
atomic< T > & as_atomic(T &t)
void __TBB_EXPORTED_METHOD internal_attach()
void process(generic_scheduler &)
Registers the worker with the arena and enters TBB scheduler dispatch loop.
void __TBB_EXPORTED_METHOD internal_enqueue(task &, intptr_t) const
unsigned hint_for_pop
Hint provided for operations with the container of starvation-resistant tasks.
void nested_arena_entry(arena *, size_t)
A functor that spawns a task.
int ref_count() const
The internal reference count.
arena * my_arena
The arena that I own (if master) or am servicing at the moment (if worker)