Intel(R) Threading Building Blocks Doxygen Documentation version 4.2.3
Loading...
Searching...
No Matches
arena.cpp
Go to the documentation of this file.
1/*
2 Copyright (c) 2005-2020 Intel Corporation
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15*/
16
17#include "tbb/global_control.h" // thread_stack_size
18
19#include "scheduler.h"
20#include "governor.h"
21#include "arena.h"
22#include "itt_notify.h"
23#include "semaphore.h"
25
26#include <functional>
27
28#if __TBB_STATISTICS_STDOUT
29#include <cstdio>
30#endif
31
32namespace tbb {
33namespace internal {
34
35#if __TBB_NUMA_SUPPORT
36class numa_binding_observer : public tbb::task_scheduler_observer {
37 int my_numa_node_id;
38 binding_handler* binding_handler_ptr;
39public:
40 numa_binding_observer( task_arena* ta, int numa_id, int num_slots )
41 : task_scheduler_observer(*ta)
42 , my_numa_node_id(numa_id)
43 , binding_handler_ptr(tbb::internal::construct_binding_handler(num_slots))
44 {}
45
46 void on_scheduler_entry( bool ) __TBB_override {
47 tbb::internal::bind_thread_to_node(
48 binding_handler_ptr, this_task_arena::current_thread_index(), my_numa_node_id);
49 }
50
51 void on_scheduler_exit( bool ) __TBB_override {
52 tbb::internal::restore_affinity_mask(binding_handler_ptr, this_task_arena::current_thread_index());
53 }
54
55 ~numa_binding_observer(){
56 tbb::internal::destroy_binding_handler(binding_handler_ptr);
57 }
58};
59
60numa_binding_observer* construct_binding_observer( tbb::interface7::task_arena* ta,
61 int numa_id, int num_slots ) {
62 numa_binding_observer* binding_observer = NULL;
63 // numa_topology initialization will be lazily performed inside nodes_count() call
64 if (numa_id >= 0 && numa_topology::nodes_count() > 1) {
65 binding_observer = new numa_binding_observer(ta, numa_id, num_slots);
66 __TBB_ASSERT(binding_observer, "Failure during NUMA binding observer allocation and construction");
67 binding_observer->observe(true);
68 }
69 return binding_observer;
70}
71
72void destroy_binding_observer( numa_binding_observer* binding_observer ) {
73 __TBB_ASSERT(binding_observer, "Trying to deallocate NULL pointer");
74 binding_observer->observe(false);
75 delete binding_observer;
76}
77#endif
78
79// put it here in order to enable compiler to inline it into arena::process and nested_arena_entry
80void generic_scheduler::attach_arena( arena* a, size_t index, bool is_master ) {
81 __TBB_ASSERT( a->my_market == my_market, NULL );
82 my_arena = a;
83 my_arena_index = index;
84 my_arena_slot = a->my_slots + index;
85 attach_mailbox( affinity_id(index+1) );
86 if ( is_master && my_inbox.is_idle_state( true ) ) {
87 // Master enters an arena with its own task to be executed. It means that master is not
88 // going to enter stealing loop and take affinity tasks.
89 my_inbox.set_is_idle( false );
90 }
91#if __TBB_TASK_GROUP_CONTEXT
92 // Context to be used by root tasks by default (if the user has not specified one).
93 if( !is_master )
94 my_dummy_task->prefix().context = a->my_default_ctx;
95#endif /* __TBB_TASK_GROUP_CONTEXT */
96#if __TBB_TASK_PRIORITY
97 // In the current implementation master threads continue processing even when
98 // there are other masters with higher priority. Only TBB worker threads are
99 // redistributed between arenas based on the latters' priority. Thus master
100 // threads use arena's top priority as a reference point (in contrast to workers
101 // that use my_market->my_global_top_priority).
102 if( is_master ) {
103 my_ref_top_priority = &a->my_top_priority;
104 my_ref_reload_epoch = &a->my_reload_epoch;
105 }
106 my_local_reload_epoch = *my_ref_reload_epoch;
107 __TBB_ASSERT( !my_offloaded_tasks, NULL );
108#endif /* __TBB_TASK_PRIORITY */
109}
110
111inline static bool occupy_slot( generic_scheduler*& slot, generic_scheduler& s ) {
112 return !slot && as_atomic( slot ).compare_and_swap( &s, NULL ) == NULL;
113}
114
115size_t arena::occupy_free_slot_in_range( generic_scheduler& s, size_t lower, size_t upper ) {
116 if ( lower >= upper ) return out_of_arena;
117 // Start search for an empty slot from the one we occupied the last time
118 size_t index = s.my_arena_index;
119 if ( index < lower || index >= upper ) index = s.my_random.get() % (upper - lower) + lower;
120 __TBB_ASSERT( index >= lower && index < upper, NULL );
121 // Find a free slot
122 for ( size_t i = index; i < upper; ++i )
123 if ( occupy_slot(my_slots[i].my_scheduler, s) ) return i;
124 for ( size_t i = lower; i < index; ++i )
125 if ( occupy_slot(my_slots[i].my_scheduler, s) ) return i;
126 return out_of_arena;
127}
128
129template <bool as_worker>
131 // Firstly, masters try to occupy reserved slots
132 size_t index = as_worker ? out_of_arena : occupy_free_slot_in_range( s, 0, my_num_reserved_slots );
133 if ( index == out_of_arena ) {
134 // Secondly, all threads try to occupy all non-reserved slots
135 index = occupy_free_slot_in_range( s, my_num_reserved_slots, my_num_slots );
136 // Likely this arena is already saturated
137 if ( index == out_of_arena )
138 return out_of_arena;
139 }
140
141 ITT_NOTIFY(sync_acquired, my_slots + index);
142 atomic_update( my_limit, (unsigned)(index + 1), std::less<unsigned>() );
143 return index;
144}
145
147 __TBB_ASSERT( is_alive(my_guard), NULL );
149 __TBB_ASSERT( s.my_innermost_running_task == s.my_dummy_task, NULL );
150 __TBB_ASSERT( s.worker_outermost_level(), NULL );
151
152 __TBB_ASSERT( my_num_slots > 1, NULL );
153
154 size_t index = occupy_free_slot</*as_worker*/true>( s );
155 if ( index == out_of_arena )
156 goto quit;
157
158 __TBB_ASSERT( index >= my_num_reserved_slots, "Workers cannot occupy reserved slots" );
159 s.attach_arena( this, index, /*is_master*/false );
160
161#if !__TBB_FP_CONTEXT
162 my_cpu_ctl_env.set_env();
163#endif
164
165#if __TBB_ARENA_OBSERVER
166 __TBB_ASSERT( !s.my_last_local_observer, "There cannot be notified local observers when entering arena" );
167 my_observers.notify_entry_observers( s.my_last_local_observer, /*worker=*/true );
168#endif /* __TBB_ARENA_OBSERVER */
169
170 // Task pool can be marked as non-empty if the worker occupies the slot left by a master.
171 if ( s.my_arena_slot->task_pool != EmptyTaskPool ) {
172 __TBB_ASSERT( s.my_inbox.is_idle_state(false), NULL );
173 s.local_wait_for_all( *s.my_dummy_task, NULL );
174 __TBB_ASSERT( s.my_inbox.is_idle_state(true), NULL );
175 }
176
177 for ( ;; ) {
178 __TBB_ASSERT( s.my_innermost_running_task == s.my_dummy_task, NULL );
179 __TBB_ASSERT( s.worker_outermost_level(), NULL );
180 __TBB_ASSERT( is_alive(my_guard), NULL );
181 __TBB_ASSERT( s.is_quiescent_local_task_pool_reset(),
182 "Worker cannot leave arena while its task pool is not reset" );
183 __TBB_ASSERT( s.my_arena_slot->task_pool == EmptyTaskPool, "Empty task pool is not marked appropriately" );
184 // This check prevents relinquishing more than necessary workers because
185 // of the non-atomicity of the decision making procedure
186 if ( is_recall_requested() )
187 break;
188 // Try to steal a task.
189 // Passing reference count is technically unnecessary in this context,
190 // but omitting it here would add checks inside the function.
191 task* t = s.receive_or_steal_task( __TBB_ISOLATION_ARG( s.my_dummy_task->prefix().ref_count, no_isolation ) );
192 if (t) {
193 // A side effect of receive_or_steal_task is that my_innermost_running_task can be set.
194 // But for the outermost dispatch loop it has to be a dummy task.
195 s.my_innermost_running_task = s.my_dummy_task;
196 s.local_wait_for_all(*s.my_dummy_task,t);
197 }
198 }
199#if __TBB_ARENA_OBSERVER
200 my_observers.notify_exit_observers( s.my_last_local_observer, /*worker=*/true );
201 s.my_last_local_observer = NULL;
202#endif /* __TBB_ARENA_OBSERVER */
203#if __TBB_TASK_PRIORITY
204 if ( s.my_offloaded_tasks )
205 orphan_offloaded_tasks( s );
206#endif /* __TBB_TASK_PRIORITY */
207#if __TBB_STATISTICS
208 ++s.my_counters.arena_roundtrips;
209 *my_slots[index].my_counters += s.my_counters;
210 s.my_counters.reset();
211#endif /* __TBB_STATISTICS */
212 __TBB_store_with_release( my_slots[index].my_scheduler, (generic_scheduler*)NULL );
213 s.my_arena_slot = 0; // detached from slot
214 s.my_inbox.detach();
215 __TBB_ASSERT( s.my_inbox.is_idle_state(true), NULL );
216 __TBB_ASSERT( s.my_innermost_running_task == s.my_dummy_task, NULL );
217 __TBB_ASSERT( s.worker_outermost_level(), NULL );
218 __TBB_ASSERT( is_alive(my_guard), NULL );
219quit:
220 // In contrast to earlier versions of TBB (before 3.0 U5) now it is possible
221 // that arena may be temporarily left unpopulated by threads. See comments in
222 // arena::on_thread_leaving() for more details.
223 on_thread_leaving<ref_worker>();
224}
225
226arena::arena ( market& m, unsigned num_slots, unsigned num_reserved_slots ) {
227 __TBB_ASSERT( !my_guard, "improperly allocated arena?" );
228 __TBB_ASSERT( sizeof(my_slots[0]) % NFS_GetLineSize()==0, "arena::slot size not multiple of cache line size" );
229 __TBB_ASSERT( (uintptr_t)this % NFS_GetLineSize()==0, "arena misaligned" );
230#if __TBB_TASK_PRIORITY
231 __TBB_ASSERT( !my_reload_epoch && !my_orphaned_tasks && !my_skipped_fifo_priority, "New arena object is not zeroed" );
232#endif /* __TBB_TASK_PRIORITY */
233 my_market = &m;
234 my_limit = 1;
235 // Two slots are mandatory: for the master, and for 1 worker (required to support starvation resistant tasks).
236 my_num_slots = num_arena_slots(num_slots);
237 my_num_reserved_slots = num_reserved_slots;
238 my_max_num_workers = num_slots-num_reserved_slots;
239 my_references = ref_external; // accounts for the master
240#if __TBB_TASK_PRIORITY
241 my_bottom_priority = my_top_priority = normalized_normal_priority;
242#endif /* __TBB_TASK_PRIORITY */
243 my_aba_epoch = m.my_arenas_aba_epoch;
244#if __TBB_ARENA_OBSERVER
245 my_observers.my_arena = this;
246#endif
247#if __TBB_PREVIEW_RESUMABLE_TASKS
248 my_co_cache.init(4 * num_slots);
249#endif
250 __TBB_ASSERT ( my_max_num_workers <= my_num_slots, NULL );
251 // Construct slots. Mark internal synchronization elements for the tools.
252 for( unsigned i = 0; i < my_num_slots; ++i ) {
253 __TBB_ASSERT( !my_slots[i].my_scheduler && !my_slots[i].task_pool, NULL );
254 __TBB_ASSERT( !my_slots[i].task_pool_ptr, NULL );
255 __TBB_ASSERT( !my_slots[i].my_task_pool_size, NULL );
256#if __TBB_PREVIEW_RESUMABLE_TASKS
257 __TBB_ASSERT( !my_slots[i].my_scheduler_is_recalled, NULL );
258#endif
259 ITT_SYNC_CREATE(my_slots + i, SyncType_Scheduler, SyncObj_WorkerTaskPool);
260 mailbox(i+1).construct();
261 ITT_SYNC_CREATE(&mailbox(i+1), SyncType_Scheduler, SyncObj_Mailbox);
262 my_slots[i].hint_for_pop = i;
263#if __TBB_PREVIEW_CRITICAL_TASKS
264 my_slots[i].hint_for_critical = i;
265#endif
266#if __TBB_STATISTICS
267 my_slots[i].my_counters = new ( NFS_Allocate(1, sizeof(statistics_counters), NULL) ) statistics_counters;
268#endif /* __TBB_STATISTICS */
269 }
270 my_task_stream.initialize(my_num_slots);
271 ITT_SYNC_CREATE(&my_task_stream, SyncType_Scheduler, SyncObj_TaskStream);
272#if __TBB_PREVIEW_CRITICAL_TASKS
273 my_critical_task_stream.initialize(my_num_slots);
274 ITT_SYNC_CREATE(&my_critical_task_stream, SyncType_Scheduler, SyncObj_CriticalTaskStream);
275#endif
276#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY
277 my_local_concurrency_mode = false;
278 my_global_concurrency_mode = false;
279#endif
280#if !__TBB_FP_CONTEXT
281 my_cpu_ctl_env.get_env();
282#endif
283}
284
285arena& arena::allocate_arena( market& m, unsigned num_slots, unsigned num_reserved_slots ) {
286 __TBB_ASSERT( sizeof(base_type) + sizeof(arena_slot) == sizeof(arena), "All arena data fields must go to arena_base" );
287 __TBB_ASSERT( sizeof(base_type) % NFS_GetLineSize() == 0, "arena slots area misaligned: wrong padding" );
288 __TBB_ASSERT( sizeof(mail_outbox) == NFS_MaxLineSize, "Mailbox padding is wrong" );
289 size_t n = allocation_size(num_arena_slots(num_slots));
290 unsigned char* storage = (unsigned char*)NFS_Allocate( 1, n, NULL );
291 // Zero all slots to indicate that they are empty
292 memset( storage, 0, n );
293 return *new( storage + num_arena_slots(num_slots) * sizeof(mail_outbox) ) arena(m, num_slots, num_reserved_slots);
294}
295
297 __TBB_ASSERT( is_alive(my_guard), NULL );
298 __TBB_ASSERT( !my_references, "There are threads in the dying arena" );
299 __TBB_ASSERT( !my_num_workers_requested && !my_num_workers_allotted, "Dying arena requests workers" );
300 __TBB_ASSERT( my_pool_state == SNAPSHOT_EMPTY || !my_max_num_workers, "Inconsistent state of a dying arena" );
301#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY
302 __TBB_ASSERT( !my_global_concurrency_mode, NULL );
303#endif
304#if !__TBB_STATISTICS_EARLY_DUMP
305 GATHER_STATISTIC( dump_arena_statistics() );
306#endif
307 poison_value( my_guard );
308 intptr_t drained = 0;
309 for ( unsigned i = 0; i < my_num_slots; ++i ) {
310 __TBB_ASSERT( !my_slots[i].my_scheduler, "arena slot is not empty" );
311 // TODO: understand the assertion and modify
312 // __TBB_ASSERT( my_slots[i].task_pool == EmptyTaskPool, NULL );
313 __TBB_ASSERT( my_slots[i].head == my_slots[i].tail, NULL ); // TODO: replace by is_quiescent_local_task_pool_empty
315#if __TBB_STATISTICS
316 NFS_Free( my_slots[i].my_counters );
317#endif /* __TBB_STATISTICS */
318 drained += mailbox(i+1).drain();
319 }
320 __TBB_ASSERT( my_task_stream.drain()==0, "Not all enqueued tasks were executed");
321#if __TBB_PREVIEW_RESUMABLE_TASKS
322 // Cleanup coroutines/schedulers cache
323 my_co_cache.cleanup();
324#endif
325#if __TBB_PREVIEW_CRITICAL_TASKS
326 __TBB_ASSERT( my_critical_task_stream.drain()==0, "Not all critical tasks were executed");
327#endif
328#if __TBB_COUNT_TASK_NODES
329 my_market->update_task_node_count( -drained );
330#endif /* __TBB_COUNT_TASK_NODES */
331 // remove an internal reference
332 my_market->release( /*is_public=*/false, /*blocking_terminate=*/false );
333#if __TBB_TASK_GROUP_CONTEXT
334 __TBB_ASSERT( my_default_ctx, "Master thread never entered the arena?" );
335 my_default_ctx->~task_group_context();
336 NFS_Free(my_default_ctx);
337#endif /* __TBB_TASK_GROUP_CONTEXT */
338#if __TBB_ARENA_OBSERVER
339 if ( !my_observers.empty() )
340 my_observers.clear();
341#endif /* __TBB_ARENA_OBSERVER */
342 void* storage = &mailbox(my_num_slots);
343 __TBB_ASSERT( my_references == 0, NULL );
344 __TBB_ASSERT( my_pool_state == SNAPSHOT_EMPTY || !my_max_num_workers, NULL );
345 this->~arena();
346#if TBB_USE_ASSERT > 1
347 memset( storage, 0, allocation_size(my_num_slots) );
348#endif /* TBB_USE_ASSERT */
349 NFS_Free( storage );
350}
351
352#if __TBB_STATISTICS
353void arena::dump_arena_statistics () {
354 statistics_counters total;
355 for( unsigned i = 0; i < my_num_slots; ++i ) {
356#if __TBB_STATISTICS_EARLY_DUMP
357 generic_scheduler* s = my_slots[i].my_scheduler;
358 if ( s )
359 *my_slots[i].my_counters += s->my_counters;
360#else
361 __TBB_ASSERT( !my_slots[i].my_scheduler, NULL );
362#endif
363 if ( i != 0 ) {
364 total += *my_slots[i].my_counters;
365 dump_statistics( *my_slots[i].my_counters, i );
366 }
367 }
368 dump_statistics( *my_slots[0].my_counters, 0 );
369#if __TBB_STATISTICS_STDOUT
370#if !__TBB_STATISTICS_TOTALS_ONLY
371 printf( "----------------------------------------------\n" );
372#endif
373 dump_statistics( total, workers_counters_total );
374 total += *my_slots[0].my_counters;
375 dump_statistics( total, arena_counters_total );
376#if !__TBB_STATISTICS_TOTALS_ONLY
377 printf( "==============================================\n" );
378#endif
379#endif /* __TBB_STATISTICS_STDOUT */
380}
381#endif /* __TBB_STATISTICS */
382
383#if __TBB_TASK_PRIORITY
384// The method inspects a scheduler to determine:
385// 1. if it has tasks that can be retrieved and executed (via the return value);
386// 2. if it has any tasks at all, including those of lower priority (via tasks_present);
387// 3. if it is able to work with enqueued tasks (via dequeuing_possible).
388inline bool arena::may_have_tasks ( generic_scheduler* s, bool& tasks_present, bool& dequeuing_possible ) {
389 if ( !s || s->my_arena != this )
390 return false;
391 dequeuing_possible |= s->worker_outermost_level();
392 if ( s->my_pool_reshuffling_pending ) {
393 // This primary task pool is nonempty and may contain tasks at the current
394 // priority level. Its owner is winnowing lower priority tasks at the moment.
395 tasks_present = true;
396 return true;
397 }
398 if ( s->my_offloaded_tasks ) {
399 tasks_present = true;
400 if ( s->my_local_reload_epoch < *s->my_ref_reload_epoch ) {
401 // This scheduler's offload area is nonempty and may contain tasks at the
402 // current priority level.
403 return true;
404 }
405 }
406 return false;
407}
408
409void arena::orphan_offloaded_tasks(generic_scheduler& s) {
410 __TBB_ASSERT( s.my_offloaded_tasks, NULL );
411 GATHER_STATISTIC( ++s.my_counters.prio_orphanings );
412 ++my_abandonment_epoch;
413 __TBB_ASSERT( s.my_offloaded_task_list_tail_link && !*s.my_offloaded_task_list_tail_link, NULL );
414 task* orphans;
415 do {
416 orphans = const_cast<task*>(my_orphaned_tasks);
417 *s.my_offloaded_task_list_tail_link = orphans;
418 } while ( as_atomic(my_orphaned_tasks).compare_and_swap(s.my_offloaded_tasks, orphans) != orphans );
419 s.my_offloaded_tasks = NULL;
420#if TBB_USE_ASSERT
421 s.my_offloaded_task_list_tail_link = NULL;
422#endif /* TBB_USE_ASSERT */
423}
424#endif /* __TBB_TASK_PRIORITY */
425
427 // Look for enqueued tasks at all priority levels
428 for ( int p = 0; p < num_priority_levels; ++p )
429 if ( !my_task_stream.empty(p) )
430 return true;
431 return false;
432}
433
435 // Check for the presence of enqueued tasks "lost" on some of
436 // priority levels because updating arena priority and switching
437 // arena into "populated" (FULL) state happen non-atomically.
438 // Imposing atomicity would require task::enqueue() to use a lock,
439 // which is unacceptable.
440 if ( has_enqueued_tasks() ) {
441 advertise_new_work<work_enqueued>();
442#if __TBB_TASK_PRIORITY
443 // update_arena_priority() expects non-zero arena::my_num_workers_requested,
444 // so must be called after advertise_new_work<work_enqueued>()
445 for ( int p = 0; p < num_priority_levels; ++p )
446 if ( !my_task_stream.empty(p) ) {
447 if ( p < my_bottom_priority || p > my_top_priority )
448 my_market->update_arena_priority(*this, p);
449 }
450#endif
451 }
452}
453
455 // TODO: rework it to return at least a hint about where a task was found; better if the task itself.
456 for(;;) {
457 pool_state_t snapshot = my_pool_state;
458 switch( snapshot ) {
459 case SNAPSHOT_EMPTY:
460 return true;
461 case SNAPSHOT_FULL: {
462 // Use unique id for "busy" in order to avoid ABA problems.
463 const pool_state_t busy = pool_state_t(&busy);
464 // Request permission to take snapshot
465 if( my_pool_state.compare_and_swap( busy, SNAPSHOT_FULL )==SNAPSHOT_FULL ) {
466 // Got permission. Take the snapshot.
467 // NOTE: This is not a lock, as the state can be set to FULL at
468 // any moment by a thread that spawns/enqueues new task.
469 size_t n = my_limit;
470 // Make local copies of volatile parameters. Their change during
471 // snapshot taking procedure invalidates the attempt, and returns
472 // this thread into the dispatch loop.
473#if __TBB_TASK_PRIORITY
474 uintptr_t reload_epoch = __TBB_load_with_acquire( my_reload_epoch );
475 intptr_t top_priority = my_top_priority;
476 // Inspect primary task pools first
477#endif /* __TBB_TASK_PRIORITY */
478 size_t k;
479 for( k=0; k<n; ++k ) {
480 if( my_slots[k].task_pool != EmptyTaskPool &&
482 {
483 // k-th primary task pool is nonempty and does contain tasks.
484 break;
485 }
486 if( my_pool_state!=busy )
487 return false; // the work was published
488 }
489 __TBB_ASSERT( k <= n, NULL );
490 bool work_absent = k == n;
491#if __TBB_PREVIEW_CRITICAL_TASKS
492 bool no_critical_tasks = my_critical_task_stream.empty(0);
493 work_absent &= no_critical_tasks;
494#endif
495#if __TBB_TASK_PRIORITY
496 // Variable tasks_present indicates presence of tasks at any priority
497 // level, while work_absent refers only to the current priority.
498 bool tasks_present = !work_absent || my_orphaned_tasks;
499 bool dequeuing_possible = false;
500 if ( work_absent ) {
501 // Check for the possibility that recent priority changes
502 // brought some tasks to the current priority level
503
504 uintptr_t abandonment_epoch = my_abandonment_epoch;
505 // Master thread's scheduler needs special handling as it
506 // may be destroyed at any moment (workers' schedulers are
507 // guaranteed to be alive while at least one thread is in arena).
508 // The lock below excludes concurrency with task group state change
509 // propagation and guarantees lifetime of the master thread.
510 the_context_state_propagation_mutex.lock();
511 work_absent = !may_have_tasks( my_slots[0].my_scheduler, tasks_present, dequeuing_possible );
512 the_context_state_propagation_mutex.unlock();
513 // The following loop is subject to data races. While k-th slot's
514 // scheduler is being examined, corresponding worker can either
515 // leave to RML or migrate to another arena.
516 // But the races are not prevented because all of them are benign.
517 // First, the code relies on the fact that worker thread's scheduler
518 // object persists until the whole library is deinitialized.
519 // Second, in the worst case the races can only cause another
520 // round of stealing attempts to be undertaken. Introducing complex
521 // synchronization into this coldest part of the scheduler's control
522 // flow does not seem to make sense because it both is unlikely to
523 // ever have any observable performance effect, and will require
524 // additional synchronization code on the hotter paths.
525 for( k = 1; work_absent && k < n; ++k ) {
526 if( my_pool_state!=busy )
527 return false; // the work was published
528 work_absent = !may_have_tasks( my_slots[k].my_scheduler, tasks_present, dequeuing_possible );
529 }
530 // Preclude premature switching arena off because of a race in the previous loop.
531 work_absent = work_absent
532 && !__TBB_load_with_acquire(my_orphaned_tasks)
533 && abandonment_epoch == my_abandonment_epoch;
534 }
535#endif /* __TBB_TASK_PRIORITY */
536 // Test and test-and-set.
537 if( my_pool_state==busy ) {
538#if __TBB_TASK_PRIORITY
539 bool no_fifo_tasks = my_task_stream.empty(top_priority);
540 work_absent = work_absent && (!dequeuing_possible || no_fifo_tasks)
541 && top_priority == my_top_priority && reload_epoch == my_reload_epoch;
542#else
543 bool no_fifo_tasks = my_task_stream.empty(0);
544 work_absent = work_absent && no_fifo_tasks;
545#endif /* __TBB_TASK_PRIORITY */
546 if( work_absent ) {
547#if __TBB_TASK_PRIORITY
548 if ( top_priority > my_bottom_priority ) {
549 if ( my_market->lower_arena_priority(*this, top_priority - 1, reload_epoch)
550 && !my_task_stream.empty(top_priority) )
551 {
552 atomic_update( my_skipped_fifo_priority, top_priority, std::less<intptr_t>());
553 }
554 }
555 else if ( !tasks_present && !my_orphaned_tasks && no_fifo_tasks ) {
556#endif /* __TBB_TASK_PRIORITY */
557 // save current demand value before setting SNAPSHOT_EMPTY,
558 // to avoid race with advertise_new_work.
559 int current_demand = (int)my_max_num_workers;
560 if( my_pool_state.compare_and_swap( SNAPSHOT_EMPTY, busy )==busy ) {
561 // This thread transitioned pool to empty state, and thus is
562 // responsible for telling the market that there is no work to do.
563 my_market->adjust_demand( *this, -current_demand );
565 return true;
566 }
567 return false;
568#if __TBB_TASK_PRIORITY
569 }
570#endif /* __TBB_TASK_PRIORITY */
571 }
572 // Undo previous transition SNAPSHOT_FULL-->busy, unless another thread undid it.
573 my_pool_state.compare_and_swap( SNAPSHOT_FULL, busy );
574 }
575 }
576 return false;
577 }
578 default:
579 // Another thread is taking a snapshot.
580 return false;
581 }
582 }
583}
584
585#if __TBB_COUNT_TASK_NODES
586intptr_t arena::workers_task_node_count() {
587 intptr_t result = 0;
588 for( unsigned i = 1; i < my_num_slots; ++i ) {
589 generic_scheduler* s = my_slots[i].my_scheduler;
590 if( s )
591 result += s->my_task_node_count;
592 }
593 return result;
594}
595#endif /* __TBB_COUNT_TASK_NODES */
596
597void arena::enqueue_task( task& t, intptr_t prio, FastRandom &random )
598{
599#if __TBB_RECYCLE_TO_ENQUEUE
600 __TBB_ASSERT( t.state()==task::allocated || t.state()==task::to_enqueue, "attempt to enqueue task with inappropriate state" );
601#else
602 __TBB_ASSERT( t.state()==task::allocated, "attempt to enqueue task that is not in 'allocated' state" );
603#endif
605 t.prefix().extra_state |= es_task_enqueued; // enqueued task marker
606
607#if TBB_USE_ASSERT
608 if( task* parent = t.parent() ) {
609 internal::reference_count ref_count = parent->prefix().ref_count;
610 __TBB_ASSERT( ref_count!=0, "attempt to enqueue task whose parent has a ref_count==0 (forgot to set_ref_count?)" );
611 __TBB_ASSERT( ref_count>0, "attempt to enqueue task whose parent has a ref_count<0" );
612 parent->prefix().extra_state |= es_ref_count_active;
613 }
614 __TBB_ASSERT(t.prefix().affinity==affinity_id(0), "affinity is ignored for enqueued tasks");
615#endif /* TBB_USE_ASSERT */
616#if __TBB_PREVIEW_CRITICAL_TASKS
617
618#if __TBB_TASK_PRIORITY
620#else
622#endif
624 if( is_critical ) {
625 // TODO: consider using of 'scheduler::handled_as_critical'
628 ITT_NOTIFY(sync_releasing, &my_critical_task_stream);
629 if( s && s->my_arena_slot ) {
630 // Scheduler is initialized and it is attached to the arena,
631 // propagate isolation level to critical task
632#if __TBB_TASK_ISOLATION
633 t.prefix().isolation = s->my_innermost_running_task->prefix().isolation;
634#endif
635 unsigned& lane = s->my_arena_slot->hint_for_critical;
636 my_critical_task_stream.push( &t, 0, tbb::internal::subsequent_lane_selector(lane) );
637 } else {
638 // Either scheduler is not initialized or it is not attached to the arena
639 // use random lane for the task
640 my_critical_task_stream.push( &t, 0, internal::random_lane_selector(random) );
641 }
642 advertise_new_work<work_spawned>();
643 return;
644 }
645#endif /* __TBB_PREVIEW_CRITICAL_TASKS */
646
647 ITT_NOTIFY(sync_releasing, &my_task_stream);
648#if __TBB_TASK_PRIORITY
649 intptr_t p = prio ? normalize_priority(priority_t(prio)) : normalized_normal_priority;
650 assert_priority_valid(p);
651#if __TBB_PREVIEW_CRITICAL_TASKS && __TBB_CPF_BUILD
652 my_task_stream.push( &t, p, internal::random_lane_selector(random) );
653#else
654 my_task_stream.push( &t, p, random );
655#endif
656 if ( p != my_top_priority )
657 my_market->update_arena_priority( *this, p );
658#else /* !__TBB_TASK_PRIORITY */
659 __TBB_ASSERT_EX(prio == 0, "the library is not configured to respect the task priority");
660#if __TBB_PREVIEW_CRITICAL_TASKS && __TBB_CPF_BUILD
661 my_task_stream.push( &t, 0, internal::random_lane_selector(random) );
662#else
663 my_task_stream.push( &t, 0, random );
664#endif
665#endif /* !__TBB_TASK_PRIORITY */
666 advertise_new_work<work_enqueued>();
667#if __TBB_TASK_PRIORITY
668 if ( p != my_top_priority )
669 my_market->update_arena_priority( *this, p );
670#endif /* __TBB_TASK_PRIORITY */
671}
672
674public:
675 nested_arena_context(generic_scheduler *s, arena* a, size_t slot_index, bool type, bool same)
676 : my_scheduler(*s), my_orig_ctx(NULL), same_arena(same) {
677 if (same_arena) {
681 } else {
682 my_orig_state = *s;
683#if __TBB_PREVIEW_RESUMABLE_TASKS
684 my_scheduler.my_properties.genuine = true;
685 my_scheduler.my_current_is_recalled = NULL;
686#endif
688 s->nested_arena_entry(a, slot_index);
689 }
690 }
692#if __TBB_TASK_GROUP_CONTEXT
693 my_scheduler.my_dummy_task->prefix().context = my_orig_ctx; // restore context of dummy task
694#endif
695 if (same_arena) {
698 } else {
700 static_cast<scheduler_state&>(my_scheduler) = my_orig_state; // restore arena settings
701#if __TBB_TASK_PRIORITY
702 my_scheduler.my_local_reload_epoch = *my_orig_state.my_ref_reload_epoch;
703#endif
705 }
706 }
707
708private:
712 const bool same_arena;
713
718#if __TBB_PREVIEW_CRITICAL_TASKS
719 my_scheduler.my_properties.has_taken_critical_task = false;
720#endif
721#if __TBB_TASK_GROUP_CONTEXT
722 // Save dummy's context and replace it by arena's context
724 my_scheduler.my_dummy_task->prefix().context = a->my_default_ctx;
725#endif
726 }
727};
728
729void generic_scheduler::nested_arena_entry(arena* a, size_t slot_index) {
730 __TBB_ASSERT( is_alive(a->my_guard), NULL );
731 __TBB_ASSERT( a!=my_arena, NULL);
732
733 // overwrite arena settings
734#if __TBB_TASK_PRIORITY
735 if ( my_offloaded_tasks )
736 my_arena->orphan_offloaded_tasks( *this );
737 my_offloaded_tasks = NULL;
738#endif /* __TBB_TASK_PRIORITY */
739 attach_arena( a, slot_index, /*is_master*/true );
740 __TBB_ASSERT( my_arena == a, NULL );
742 // TODO? ITT_NOTIFY(sync_acquired, a->my_slots + index);
743 // TODO: it requires market to have P workers (not P-1)
744 // TODO: a preempted worker should be excluded from assignment to other arenas e.g. my_slack--
745 if( !is_worker() && slot_index >= my_arena->my_num_reserved_slots )
746 my_arena->my_market->adjust_demand(*my_arena, -1);
747#if __TBB_ARENA_OBSERVER
748 my_last_local_observer = 0; // TODO: try optimize number of calls
749 my_arena->my_observers.notify_entry_observers( my_last_local_observer, /*worker=*/false );
750#endif
751#if __TBB_PREVIEW_RESUMABLE_TASKS
752 my_wait_task = NULL;
753#endif
754}
755
757#if __TBB_ARENA_OBSERVER
758 my_arena->my_observers.notify_exit_observers( my_last_local_observer, /*worker=*/false );
759#endif /* __TBB_ARENA_OBSERVER */
760#if __TBB_TASK_PRIORITY
761 if ( my_offloaded_tasks )
762 my_arena->orphan_offloaded_tasks( *this );
763#endif
764 if( !is_worker() && my_arena_index >= my_arena->my_num_reserved_slots )
765 my_arena->my_market->adjust_demand(*my_arena, 1);
766 // Free the master slot.
767 __TBB_ASSERT(my_arena->my_slots[my_arena_index].my_scheduler, "A slot is already empty");
769 my_arena->my_exit_monitors.notify_one(); // do not relax!
770}
771
773 my_dummy_task->prefix().ref_count++; // prevents exit from local_wait_for_all when local work is done enforcing the stealing
774 while( my_arena->my_pool_state != arena::SNAPSHOT_EMPTY )
777}
778
779#if __TBB_PREVIEW_RESUMABLE_TASKS
780class resume_task : public task {
781 generic_scheduler& my_target;
782public:
783 resume_task(generic_scheduler& target) : my_target(target) {}
784 task* execute() __TBB_override {
785 generic_scheduler* s = governor::local_scheduler_if_initialized();
786 __TBB_ASSERT(s, NULL);
787 if (s->prepare_resume(my_target)) {
788 s->resume(my_target);
789 } else {
790 __TBB_ASSERT(prefix().state == task::executing, NULL);
791 // Request the dispatch loop to exit (because we in a coroutine on the outermost level).
792 prefix().state = task::to_resume;
793 }
794 return NULL;
795 }
796};
797
798static generic_scheduler& create_coroutine(generic_scheduler& curr) {
799 // We may have some coroutines cached
800 generic_scheduler* co_sched = curr.my_arena->my_co_cache.pop();
801 if (!co_sched) {
802 // TODO: avoid setting/unsetting the scheduler.
804 co_sched = generic_scheduler::create_worker(*curr.my_market, curr.my_arena_index, /* genuine = */ false);
806 // Prepare newly created scheduler
807 co_sched->my_arena = curr.my_arena;
808 }
809 // Prepare scheduler (general)
810 co_sched->my_dummy_task->prefix().context = co_sched->my_arena->my_default_ctx;
811 // Prolong the arena's lifetime until all coroutines is alive
812 // (otherwise the arena can be destroyed while some tasks are suspended).
813 co_sched->my_arena->my_references += arena::ref_external;
814 return *co_sched;
815}
816
817void internal_suspend(void* suspend_callback, void* user_callback) {
818 generic_scheduler& s = *governor::local_scheduler();
819 __TBB_ASSERT(s.my_arena_slot->my_scheduler_is_recalled != NULL, NULL);
820 bool is_recalled = *s.my_arena_slot->my_scheduler_is_recalled;
821 generic_scheduler& target = is_recalled ? *s.my_arena_slot->my_scheduler : create_coroutine(s);
822
823 generic_scheduler::callback_t callback = {
824 (generic_scheduler::suspend_callback_t)suspend_callback, user_callback, &s };
825 target.set_post_resume_action(generic_scheduler::PRA_CALLBACK, &callback);
826 s.resume(target);
827}
828
829void internal_resume(task::suspend_point tag) {
830 generic_scheduler& s = *static_cast<generic_scheduler*>(tag);
831 task* t = new(&s.allocate_task(sizeof(resume_task), __TBB_CONTEXT_ARG(NULL, s.my_dummy_task->context()))) resume_task(s);
832 make_critical(*t);
833
834 // TODO: remove this work-around
835 // Prolong the arena's lifetime until all coroutines is alive
836 // (otherwise the arena can be destroyed while some tasks are suspended).
837 arena& a = *s.my_arena;
838 a.my_references += arena::ref_external;
839
840 a.my_critical_task_stream.push(t, 0, tbb::internal::random_lane_selector(s.my_random));
841 // Do not access 's' after that point.
842 a.advertise_new_work<arena::work_spawned>();
843
844 // Release our reference to my_arena.
845 a.on_thread_leaving<arena::ref_external>();
846}
847
848task::suspend_point internal_current_suspend_point() {
850}
851#endif /* __TBB_PREVIEW_RESUMABLE_TASKS */
852
853} // namespace internal
854} // namespace tbb
855
856#include "scheduler_utility.h"
857#include "tbb/task_arena.h" // task_arena_base
858
859namespace tbb {
860namespace interface7 {
861namespace internal {
862
865 if( my_max_concurrency < 1 )
866#if __TBB_NUMA_SUPPORT
867 my_max_concurrency = tbb::internal::numa_topology::default_concurrency(numa_id());
868#else /*__TBB_NUMA_SUPPORT*/
870#endif /*__TBB_NUMA_SUPPORT*/
871 __TBB_ASSERT( my_master_slots <= (unsigned)my_max_concurrency, "Number of slots reserved for master should not exceed arena concurrency");
873 // add an internal market reference; a public reference was added in create_arena
874 market &m = market::global_market( /*is_public=*/false );
875 // allocate default context for task_arena
876#if __TBB_TASK_GROUP_CONTEXT
877 new_arena->my_default_ctx = new ( NFS_Allocate(1, sizeof(task_group_context), NULL) )
879#if __TBB_FP_CONTEXT
880 new_arena->my_default_ctx->capture_fp_settings();
881#endif
882#endif /* __TBB_TASK_GROUP_CONTEXT */
883 // threads might race to initialize the arena
884 if(as_atomic(my_arena).compare_and_swap(new_arena, NULL) != NULL) {
885 __TBB_ASSERT(my_arena, NULL); // another thread won the race
886 // release public market reference
887 m.release( /*is_public=*/true, /*blocking_terminate=*/false );
888 new_arena->on_thread_leaving<arena::ref_external>(); // destroy unneeded arena
889#if __TBB_TASK_GROUP_CONTEXT
890 spin_wait_while_eq(my_context, (task_group_context*)NULL);
891#endif /*__TBB_TASK_GROUP_CONTEXT*/
892#if __TBB_TASK_GROUP_CONTEXT || __TBB_NUMA_SUPPORT
893 } else {
894#if __TBB_NUMA_SUPPORT
895 my_arena->my_numa_binding_observer = tbb::internal::construct_binding_observer(
896 static_cast<task_arena*>(this), numa_id(), my_arena->my_num_slots);
897#endif /*__TBB_NUMA_SUPPORT*/
898#if __TBB_TASK_GROUP_CONTEXT
899 new_arena->my_default_ctx->my_version_and_traits |= my_version_and_traits & exact_exception_flag;
900 as_atomic(my_context) = new_arena->my_default_ctx;
901#endif /*__TBB_TASK_GROUP_CONTEXT*/
902 }
903#endif /*__TBB_TASK_GROUP_CONTEXT || __TBB_NUMA_SUPPORT*/
904
905 // TODO: should it trigger automatic initialization of this thread?
907}
908
910 if( my_arena ) {// task_arena was initialized
911#if __TBB_NUMA_SUPPORT
912 if( my_arena->my_numa_binding_observer != NULL ) {
913 tbb::internal::destroy_binding_observer(my_arena->my_numa_binding_observer);
914 my_arena->my_numa_binding_observer = NULL;
915 }
916#endif /*__TBB_NUMA_SUPPORT*/
917 my_arena->my_market->release( /*is_public=*/true, /*blocking_terminate=*/false );
919 my_arena = 0;
920#if __TBB_TASK_GROUP_CONTEXT
921 my_context = 0;
922#endif
923 }
924}
925
927 __TBB_ASSERT(!my_arena, NULL);
928 generic_scheduler* s = governor::local_scheduler_if_initialized();
929 if( s && s->my_arena ) {
930 // There is an active arena to attach to.
931 // It's still used by s, so won't be destroyed right away.
932 my_arena = s->my_arena;
933 __TBB_ASSERT( my_arena->my_references > 0, NULL );
934 my_arena->my_references += arena::ref_external;
935#if __TBB_TASK_GROUP_CONTEXT
936 my_context = my_arena->my_default_ctx;
938#endif
939 my_master_slots = my_arena->my_num_reserved_slots;
940 my_max_concurrency = my_master_slots + my_arena->my_max_num_workers;
942 // increases market's ref count for task_arena
943 market::global_market( /*is_public=*/true );
944 }
945}
946
947void task_arena_base::internal_enqueue( task& t, intptr_t prio ) const {
948 __TBB_ASSERT(my_arena, NULL);
949 generic_scheduler* s = governor::local_scheduler_weak(); // scheduler is only needed for FastRandom instance
950 __TBB_ASSERT(s, "Scheduler is not initialized"); // we allocated a task so can expect the scheduler
951#if __TBB_TASK_GROUP_CONTEXT
952 // Is there a better place for checking the state of my_default_ctx?
953 __TBB_ASSERT(!(my_arena->my_default_ctx == t.prefix().context && my_arena->my_default_ctx->is_group_execution_cancelled()),
954 "The task will not be executed because default task_group_context of task_arena is cancelled. Has previously enqueued task thrown an exception?");
955#endif
956 my_arena->enqueue_task( t, prio, s->my_random );
957}
958
959class delegated_task : public task {
960 internal::delegate_base & my_delegate;
961 concurrent_monitor & my_monitor;
962 task * my_root;
963 task* execute() __TBB_override {
964 generic_scheduler& s = *(generic_scheduler*)prefix().owner;
965 __TBB_ASSERT(s.outermost_level(), "expected to be enqueued and received on the outermost level");
966 struct outermost_context : internal::no_copy {
967 delegated_task * t;
968 generic_scheduler & s;
969 task * orig_dummy;
970 task_group_context * orig_ctx;
971 scheduler_properties orig_props;
972 outermost_context(delegated_task *_t, generic_scheduler &_s)
973 : t(_t), s(_s), orig_dummy(s.my_dummy_task), orig_props(s.my_properties) {
974 __TBB_ASSERT(s.my_innermost_running_task == t, NULL);
975#if __TBB_TASK_GROUP_CONTEXT
976 orig_ctx = t->prefix().context;
977 t->prefix().context = s.my_arena->my_default_ctx;
978#endif
979 // Mimics outermost master
980 s.my_dummy_task = t;
981 s.my_properties.type = scheduler_properties::master;
982 }
983 ~outermost_context() {
984#if __TBB_TASK_GROUP_CONTEXT
985 // Restore context for sake of registering potential exception
986 t->prefix().context = orig_ctx;
987#endif
988 // Restore scheduler state
989 s.my_properties = orig_props;
990 s.my_dummy_task = orig_dummy;
991 }
992 } scope(this, s);
993 my_delegate();
994 return NULL;
995 }
996 ~delegated_task() {
997 // potential exception was already registered. It must happen before the notification
998 __TBB_ASSERT(my_root->ref_count() == 2, NULL);
999 task_prefix& prefix = my_root->prefix();
1000#if __TBB_PREVIEW_RESUMABLE_TASKS
1001 reference_count old_ref_count = __TBB_FetchAndStoreW(&prefix.ref_count, 1);
1002 // Check if the scheduler was abandoned.
1003 if (old_ref_count == internal::abandon_flag + 2) {
1004 __TBB_ASSERT(prefix.abandoned_scheduler, NULL);
1005 // The wait has been completed. Spawn a resume task.
1006 tbb::task::resume(prefix.abandoned_scheduler);
1007 }
1008#else
1009 __TBB_store_with_release(prefix.ref_count, 1); // must precede the wakeup
1010#endif
1011 my_monitor.notify(*this); // do not relax, it needs a fence!
1012 }
1013public:
1014 delegated_task( internal::delegate_base & d, concurrent_monitor & s, task * t )
1015 : my_delegate(d), my_monitor(s), my_root(t) {}
1016 // predicate for concurrent_monitor notification
1017 bool operator()(uintptr_t ctx) const { return (void*)ctx == (void*)&my_delegate; }
1018};
1019
1020void task_arena_base::internal_execute(internal::delegate_base& d) const {
1021 __TBB_ASSERT(my_arena, NULL);
1022 generic_scheduler* s = governor::local_scheduler_weak();
1023 __TBB_ASSERT(s, "Scheduler is not initialized");
1024
1025 bool same_arena = s->my_arena == my_arena;
1026 size_t index1 = s->my_arena_index;
1027 if (!same_arena) {
1028 index1 = my_arena->occupy_free_slot</* as_worker*/false>(*s);
1029 if (index1 == arena::out_of_arena) {
1030
1031#if __TBB_USE_OPTIONAL_RTTI
1032 // Workaround for the bug inside graph. If the thread can not occupy arena slot during task_arena::execute()
1033 // and all aggregator operations depend on this task completion (all other threads are inside arena already)
1034 // deadlock appears, because enqueued task will never enter arena.
1035 // Workaround: check if the task came from graph via RTTI (casting to graph::spawn_functor)
1036 // and enqueue this task with non-blocking internal_enqueue method.
1037 // TODO: have to change behaviour later in next GOLD release (maybe to add new library entry point - try_execute)
1039 internal::delegated_function< graph_funct, void >* deleg_funct =
1040 dynamic_cast< internal::delegated_function< graph_funct, void>* >(&d);
1041
1042 if (deleg_funct) {
1044 internal::function_task< internal::strip< graph_funct >::type >
1045 (internal::forward< graph_funct >(deleg_funct->my_func)), 0);
1046 return;
1047 } else {
1048#endif /* __TBB_USE_OPTIONAL_RTTI */
1049 concurrent_monitor::thread_context waiter;
1050#if __TBB_TASK_GROUP_CONTEXT
1052#if __TBB_FP_CONTEXT
1053 exec_context.copy_fp_settings(*my_context);
1054#endif
1055#endif
1056 auto_empty_task root(__TBB_CONTEXT_ARG(s, &exec_context));
1057 root.prefix().ref_count = 2;
1059 delegated_task(d, my_arena->my_exit_monitors, &root),
1060 0, s->my_random); // TODO: priority?
1061 size_t index2 = arena::out_of_arena;
1062 do {
1063 my_arena->my_exit_monitors.prepare_wait(waiter, (uintptr_t)&d);
1064 if (__TBB_load_with_acquire(root.prefix().ref_count) < 2) {
1065 my_arena->my_exit_monitors.cancel_wait(waiter);
1066 break;
1067 }
1068 index2 = my_arena->occupy_free_slot</*as_worker*/false>(*s);
1069 if (index2 != arena::out_of_arena) {
1070 my_arena->my_exit_monitors.cancel_wait(waiter);
1071 nested_arena_context scope(s, my_arena, index2, scheduler_properties::master, same_arena);
1072 s->local_wait_for_all(root, NULL);
1073#if TBB_USE_EXCEPTIONS
1074 __TBB_ASSERT(!exec_context.my_exception, NULL); // exception can be thrown above, not deferred
1075#endif
1076 __TBB_ASSERT(root.prefix().ref_count == 0, NULL);
1077 break;
1078 }
1079 my_arena->my_exit_monitors.commit_wait(waiter);
1080 } while (__TBB_load_with_acquire(root.prefix().ref_count) == 2);
1081 if (index2 == arena::out_of_arena) {
1082 // notify a waiting thread even if this thread did not enter arena,
1083 // in case it was woken by a leaving thread but did not need to enter
1084 my_arena->my_exit_monitors.notify_one(); // do not relax!
1085 }
1086#if TBB_USE_EXCEPTIONS
1087 // process possible exception
1088 if (task_group_context::exception_container_type *pe = exec_context.my_exception)
1089 TbbRethrowException(pe);
1090#endif
1091 return;
1092#if __TBB_USE_OPTIONAL_RTTI
1093 } // if task came from graph
1094#endif
1095 } // if (index1 == arena::out_of_arena)
1096 } // if (!same_arena)
1097
1098 context_guard_helper</*report_tasks=*/false> context_guard;
1099 context_guard.set_ctx(__TBB_CONTEXT_ARG1(my_context));
1100#if TBB_USE_EXCEPTIONS
1101 try {
1102#endif
1103 //TODO: replace dummy tasks for workers as well to avoid using of the_dummy_context
1104 nested_arena_context scope(s, my_arena, index1, scheduler_properties::master, same_arena);
1105 d();
1106#if TBB_USE_EXCEPTIONS
1107 }
1108 catch (...) {
1109 context_guard.restore_default(); // TODO: is it needed on Windows?
1111 else {
1112 task_group_context exception_container(task_group_context::isolated,
1114 exception_container.register_pending_exception();
1115 __TBB_ASSERT(exception_container.my_exception, NULL);
1116 TbbRethrowException(exception_container.my_exception);
1117 }
1118 }
1119#endif
1120}
1121
1122// this wait task is a temporary approach to wait for arena emptiness for masters without slots
1123// TODO: it will be rather reworked for one source of notification from is_out_of_work
1124class wait_task : public task {
1125 binary_semaphore & my_signal;
1126 task* execute() __TBB_override {
1127 generic_scheduler* s = governor::local_scheduler_if_initialized();
1128 __TBB_ASSERT( s, NULL );
1129 __TBB_ASSERT( s->outermost_level(), "The enqueued task can be processed only on outermost level" );
1130 if ( s->is_worker() ) {
1131 __TBB_ASSERT( s->my_innermost_running_task == this, NULL );
1132 // Mimic worker on outermost level to run remaining tasks
1133 s->my_innermost_running_task = s->my_dummy_task;
1134 s->local_wait_for_all( *s->my_dummy_task, NULL );
1135 s->my_innermost_running_task = this;
1136 } else s->my_arena->is_out_of_work(); // avoids starvation of internal_wait: issuing this task makes arena full
1137 my_signal.V();
1138 return NULL;
1139 }
1140public:
1141 wait_task ( binary_semaphore & sema ) : my_signal(sema) {}
1142};
1143
1144void task_arena_base::internal_wait() const {
1145 __TBB_ASSERT(my_arena, NULL);
1146 generic_scheduler* s = governor::local_scheduler_weak();
1147 __TBB_ASSERT(s, "Scheduler is not initialized");
1148 __TBB_ASSERT(s->my_arena != my_arena || s->my_arena_index == 0, "task_arena::wait_until_empty() is not supported within a worker context" );
1149 if( s->my_arena == my_arena ) {
1150 //unsupported, but try do something for outermost master
1151 __TBB_ASSERT(s->master_outermost_level(), "unsupported");
1152 if( !s->my_arena_index )
1153 while( my_arena->num_workers_active() )
1154 s->wait_until_empty();
1155 } else for(;;) {
1156 while( my_arena->my_pool_state != arena::SNAPSHOT_EMPTY ) {
1157 if( !__TBB_load_with_acquire(my_arena->my_slots[0].my_scheduler) // TODO TEMP: one master, make more masters
1158 && as_atomic(my_arena->my_slots[0].my_scheduler).compare_and_swap(s, NULL) == NULL ) {
1159 nested_arena_context a(s, my_arena, 0, scheduler_properties::worker, false);
1160 s->wait_until_empty();
1161 } else {
1162 binary_semaphore waiter; // TODO: replace by a single event notification from is_out_of_work
1163 internal_enqueue( *new( task::allocate_root(__TBB_CONTEXT_ARG1(*my_context)) ) wait_task(waiter), 0 ); // TODO: priority?
1164 waiter.P(); // TODO: concurrent_monitor
1165 }
1166 }
1167 if( !my_arena->num_workers_active() && !my_arena->my_slots[0].my_scheduler) // no activity
1168 break; // spin until workers active but avoid spinning in a worker
1169 __TBB_Yield(); // wait until workers and master leave
1170 }
1171}
1172
1174 generic_scheduler* s = governor::local_scheduler_if_initialized();
1175 return s? int(s->my_arena_index) : -1;
1176}
1177
1178#if __TBB_TASK_ISOLATION
1179class isolation_guard : tbb::internal::no_copy {
1180 isolation_tag &guarded;
1181 isolation_tag previous_value;
1182public:
1183 isolation_guard( isolation_tag &isolation ) : guarded( isolation ), previous_value( isolation ) {}
1184 ~isolation_guard() {
1185 guarded = previous_value;
1186 }
1187};
1188
1189void isolate_within_arena( delegate_base& d, intptr_t isolation ) {
1190 // TODO: Decide what to do if the scheduler is not initialized. Is there a use case for it?
1191 generic_scheduler* s = governor::local_scheduler_weak();
1192 __TBB_ASSERT( s, "this_task_arena::isolate() needs an initialized scheduler" );
1193 // Theoretically, we can keep the current isolation in the scheduler; however, it makes sense to store it in innermost
1194 // running task because it can in principle be queried via task::self().
1195 isolation_tag& current_isolation = s->my_innermost_running_task->prefix().isolation;
1196 // We temporarily change the isolation tag of the currently running task. It will be restored in the destructor of the guard.
1197 isolation_guard guard( current_isolation );
1198 current_isolation = isolation? isolation : reinterpret_cast<isolation_tag>(&d);
1199 d();
1200}
1201#endif /* __TBB_TASK_ISOLATION */
1202
1203int task_arena_base::internal_max_concurrency(const task_arena *ta) {
1204 arena* a = NULL;
1205 if( ta ) // for special cases of ta->max_concurrency()
1206 a = ta->my_arena;
1207 else if( generic_scheduler* s = governor::local_scheduler_if_initialized() )
1208 a = s->my_arena; // the current arena if any
1209
1210 if( a ) { // Get parameters from the arena
1211 __TBB_ASSERT( !ta || ta->my_max_concurrency==1, NULL );
1212 return a->my_num_reserved_slots + a->my_max_num_workers;
1213 } else {
1214 __TBB_ASSERT( !ta || ta->my_max_concurrency==automatic, NULL );
1216 }
1217}
1218} // tbb::interfaceX::internal
1219} // tbb::interfaceX
1220} // tbb
#define __TBB_Yield()
Definition ibm_aix51.h:44
#define __TBB_ASSERT_EX(predicate, comment)
"Extended" version is useful to suppress warnings if a variable is only used with an assert
Definition tbb_stddef.h:167
#define __TBB_ASSERT(predicate, comment)
No-op version of __TBB_ASSERT.
Definition tbb_stddef.h:165
#define __TBB_override
Definition tbb_stddef.h:240
#define ITT_SYNC_CREATE(obj, type, name)
Definition itt_notify.h:115
#define ITT_NOTIFY(name, obj)
Definition itt_notify.h:112
#define EmptyTaskPool
Definition scheduler.h:46
#define __TBB_CONTEXT_ARG(arg1, context)
#define __TBB_CONTEXT_ARG1(context)
#define __TBB_ISOLATION_ARG(arg1, isolation)
#define poison_value(g)
#define GATHER_STATISTIC(x)
void const char const char int ITT_FORMAT __itt_group_sync s
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task * task
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id parent
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p sync_releasing
void const char const char int ITT_FORMAT __itt_group_sync p
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain ITT_FORMAT p const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_id __itt_string_handle __itt_metadata_type size_t void ITT_FORMAT p const __itt_domain __itt_id __itt_string_handle const wchar_t size_t ITT_FORMAT lu const __itt_domain __itt_id __itt_relation __itt_id ITT_FORMAT p const wchar_t int ITT_FORMAT __itt_group_mark d __itt_event ITT_FORMAT __itt_group_mark d void const wchar_t const wchar_t int ITT_FORMAT __itt_group_sync __itt_group_fsync x void const wchar_t int const wchar_t int int ITT_FORMAT __itt_group_sync __itt_group_fsync x void ITT_FORMAT __itt_group_sync __itt_group_fsync p void ITT_FORMAT __itt_group_sync __itt_group_fsync p void size_t ITT_FORMAT lu no args __itt_obj_prop_t __itt_obj_state_t ITT_FORMAT d const char ITT_FORMAT s const char ITT_FORMAT s __itt_frame ITT_FORMAT p __itt_counter ITT_FORMAT p __itt_counter unsigned long long ITT_FORMAT lu __itt_counter unsigned long long ITT_FORMAT lu __itt_counter __itt_clock_domain unsigned long long void ITT_FORMAT p const wchar_t ITT_FORMAT S __itt_mark_type const wchar_t ITT_FORMAT S __itt_mark_type const char ITT_FORMAT s __itt_mark_type ITT_FORMAT d __itt_caller ITT_FORMAT p __itt_caller ITT_FORMAT p no args const __itt_domain __itt_clock_domain unsigned long long __itt_id ITT_FORMAT lu const __itt_domain __itt_clock_domain unsigned long long __itt_id __itt_id void ITT_FORMAT p const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_clock_domain unsigned long long __itt_id __itt_string_handle __itt_scope scope
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain ITT_FORMAT p const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_id __itt_string_handle __itt_metadata_type size_t void ITT_FORMAT p const __itt_domain __itt_id __itt_string_handle const wchar_t size_t ITT_FORMAT lu const __itt_domain __itt_id head
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain ITT_FORMAT p const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_id __itt_string_handle __itt_metadata_type size_t void ITT_FORMAT p const __itt_domain __itt_id __itt_string_handle const wchar_t size_t ITT_FORMAT lu const __itt_domain __itt_id __itt_relation __itt_id ITT_FORMAT p const wchar_t int ITT_FORMAT __itt_group_mark d int
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain ITT_FORMAT p const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_id __itt_string_handle __itt_metadata_type size_t void ITT_FORMAT p const __itt_domain __itt_id __itt_string_handle const wchar_t size_t ITT_FORMAT lu const __itt_domain __itt_id __itt_relation __itt_id tail
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain ITT_FORMAT p const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_id __itt_string_handle __itt_metadata_type type
void *__TBB_EXPORTED_FUNC NFS_Allocate(size_t n_element, size_t element_size, void *hint)
Allocate memory on cache/sector line boundary.
const size_t NFS_MaxLineSize
Compile-time constant that is upper bound on cache line/sector size.
Definition tbb_stddef.h:216
void __TBB_EXPORTED_FUNC NFS_Free(void *)
Free memory allocated by NFS_Allocate.
size_t __TBB_EXPORTED_FUNC NFS_GetLineSize()
Cache/sector line size.
The graph class.
priority_t
Definition task.h:317
intptr_t isolation_tag
A tag for task isolation.
Definition task.h:143
T __TBB_load_with_acquire(const volatile T &location)
intptr_t reference_count
A reference count.
Definition task.h:131
bool is_critical(task &t)
Definition task.h:1014
unsigned short affinity_id
An id as used for specifying affinity.
Definition task.h:139
@ es_task_enqueued
Tag for enqueued tasks.
@ es_ref_count_active
Set if ref_count might be changed by another thread. Used for debugging.
T1 atomic_update(tbb::atomic< T1 > &dst, T2 newValue, Pred compare)
Atomically replaces value of dst with newValue if they satisfy condition of compare predicate.
Definition tbb_misc.h:186
static bool occupy_slot(generic_scheduler *&slot, generic_scheduler &s)
Definition arena.cpp:111
static const int priority_critical
Definition task.h:313
void make_critical(task &t)
Definition task.h:1013
atomic< T > & as_atomic(T &t)
Definition atomic.h:572
T __TBB_load_relaxed(const volatile T &location)
void create_coroutine(coroutine_type &c, size_t stack_size, void *arg)
Definition co_context.h:154
const isolation_tag no_isolation
Definition task.h:144
static const intptr_t num_priority_levels
void __TBB_store_with_release(volatile T &location, V value)
void spin_wait_while_eq(const volatile T &location, U value)
Spin WHILE the value of the variable is equal to a given value.
void __TBB_EXPORTED_FUNC isolate_within_arena(delegate_base &d, intptr_t isolation=0)
unsigned char extra_state
Miscellaneous state that is not directly visible to users, stored as a byte for compactness.
Definition task.h:292
isolation_tag isolation
The tag used for task isolation.
Definition task.h:220
__TBB_atomic reference_count ref_count
Reference count used for synchronization.
Definition task.h:274
unsigned char state
A task::state_type, stored as a byte for compactness.
Definition task.h:283
task_group_context * context
Shared context that is used to communicate asynchronous state changes.
Definition task.h:230
affinity_id affinity
Definition task.h:294
Used to form groups of tasks.
Definition task.h:358
internal::tbb_exception_ptr exception_container_type
Definition task.h:367
uintptr_t my_version_and_traits
Version for run-time checks and behavioral traits of the context.
Definition task.h:446
Base class for user-defined tasks.
Definition task.h:615
task * parent() const
task on whose behalf this task is working, or NULL if this is a root.
Definition task.h:865
@ allocated
task object is freshly allocated or recycled.
Definition task.h:643
@ ready
task is in ready pool, or is going to be put there, or was just taken off.
Definition task.h:641
state_type state() const
Current execution state.
Definition task.h:912
static internal::allocate_root_proxy allocate_root()
Returns proxy for overloaded new that allocates a root task.
Definition task.h:663
internal::task_prefix & prefix(internal::version_tag *=NULL) const
Get reference to corresponding task_prefix.
Definition task.h:1002
static const int automatic
Typedef for number of threads that is automatic.
Definition task_arena.h:205
void __TBB_EXPORTED_METHOD internal_wait() const
intptr_t my_version_and_traits
Special settings.
Definition task_arena.h:134
static int __TBB_EXPORTED_FUNC internal_max_concurrency(const task_arena *)
void __TBB_EXPORTED_METHOD internal_enqueue(task &, intptr_t) const
internal::arena * my_arena
NULL if not currently initialized.
Definition task_arena.h:120
task_group_context * my_context
default context of the arena
Definition task_arena.h:124
void __TBB_EXPORTED_METHOD internal_attach()
unsigned my_master_slots
Reserved master slots.
Definition task_arena.h:131
void __TBB_EXPORTED_METHOD internal_execute(delegate_base &) const
int my_max_concurrency
Concurrency level for deferred initialization.
Definition task_arena.h:128
void __TBB_EXPORTED_METHOD internal_initialize()
static int __TBB_EXPORTED_FUNC internal_current_slot()
void __TBB_EXPORTED_METHOD internal_terminate()
Base class for types that should not be copied or assigned.
Definition tbb_stddef.h:330
generic_scheduler & my_scheduler
Definition arena.cpp:709
task_group_context * my_orig_ctx
Definition arena.cpp:711
void mimic_outermost_level(arena *a, bool type)
Definition arena.cpp:714
nested_arena_context(generic_scheduler *s, arena *a, size_t slot_index, bool type, bool same)
Definition arena.cpp:675
static const pool_state_t SNAPSHOT_EMPTY
No tasks to steal since last snapshot was taken.
Definition arena.h:318
size_t occupy_free_slot_in_range(generic_scheduler &s, size_t lower, size_t upper)
Tries to occupy a slot in the specified range.
Definition arena.cpp:115
static const unsigned ref_external
Reference increment values for externals and workers.
Definition arena.h:327
unsigned num_workers_active() const
The number of workers active in the arena.
Definition arena.h:334
size_t occupy_free_slot(generic_scheduler &s)
Tries to occupy a slot in the arena. On success, returns the slot index; if no slot is available,...
Definition arena.cpp:130
void free_arena()
Completes arena shutdown, destructs and deallocates it.
Definition arena.cpp:296
static int unsigned num_arena_slots(unsigned num_slots)
Definition arena.h:296
void enqueue_task(task &, intptr_t, FastRandom &)
enqueue a task into starvation-resistance queue
Definition arena.cpp:597
arena_slot my_slots[1]
Definition arena.h:390
static const pool_state_t SNAPSHOT_FULL
At least one task has been offered for stealing since the last snapshot started.
Definition arena.h:321
bool is_recall_requested() const
Check if the recall is requested by the market.
Definition arena.h:339
void restore_priority_if_need()
If enqueued tasks found, restore arena priority and task presence status.
Definition arena.cpp:434
static int allocation_size(unsigned num_slots)
Definition arena.h:300
bool is_out_of_work()
Check if there is job anywhere in arena.
Definition arena.cpp:454
static const size_t out_of_arena
Definition arena.h:382
mail_outbox & mailbox(affinity_id id)
Get reference to mailbox corresponding to given affinity_id.
Definition arena.h:305
uintptr_t pool_state_t
Definition arena.h:315
arena(market &, unsigned max_num_workers, unsigned num_reserved_slots)
Constructor.
Definition arena.cpp:226
bool has_enqueued_tasks()
Check for the presence of enqueued tasks at all priority levels.
Definition arena.cpp:426
static arena & allocate_arena(market &, unsigned num_slots, unsigned num_reserved_slots)
Allocate an instance of arena.
Definition arena.cpp:285
void on_thread_leaving()
Notification that worker or master leaves its arena.
Definition arena.h:394
void process(generic_scheduler &)
Registers the worker with the arena and enters TBB scheduler dispatch loop.
Definition arena.cpp:146
static bool is_set(generic_scheduler *s)
Used to check validity of the local scheduler TLS contents.
Definition governor.cpp:120
static generic_scheduler * local_scheduler_weak()
Definition governor.h:134
static generic_scheduler * local_scheduler_if_initialized()
Definition governor.h:139
static generic_scheduler * local_scheduler()
Obtain the thread-local instance of the TBB scheduler.
Definition governor.h:129
static unsigned default_num_threads()
Definition governor.h:84
static void assume_scheduler(generic_scheduler *s)
Temporarily set TLS slot to the given scheduler.
Definition governor.cpp:116
static void one_time_init()
Definition governor.cpp:156
Class representing where mail is put.
Definition mailbox.h:99
intptr_t drain()
Drain the mailbox.
Definition mailbox.h:179
void construct()
Construct *this as a mailbox from zeroed memory.
Definition mailbox.h:169
void set_is_idle(bool value)
Indicate whether thread that reads this mailbox is idle.
Definition mailbox.h:222
bool is_idle_state(bool value) const
Indicate whether thread that reads this mailbox is idle.
Definition mailbox.h:229
uintptr_t my_arenas_aba_epoch
ABA prevention marker to assign to newly created arenas.
Definition market.h:143
static market & global_market(bool is_public, unsigned max_num_workers=0, size_t stack_size=0)
Factory method creating new market object.
Definition market.cpp:96
static arena * create_arena(int num_slots, int num_reserved_slots, size_t stack_size)
Creates an arena object.
Definition market.cpp:308
bool type
Indicates that a scheduler acts as a master or a worker.
Definition scheduler.h:54
bool outermost
Indicates that a scheduler is on outermost level.
Definition scheduler.h:57
size_t my_arena_index
Index of the arena slot the scheduler occupies now, or occupied last time.
Definition scheduler.h:79
arena * my_arena
The arena that I own (if master) or am servicing at the moment (if worker)
Definition scheduler.h:85
arena_slot * my_arena_slot
Pointer to the slot in the arena we own at the moment.
Definition scheduler.h:82
task * my_innermost_running_task
Innermost task whose task::execute() is running. A dummy task on the outermost level.
Definition scheduler.h:88
scheduler_properties my_properties
Definition scheduler.h:101
Work stealing task scheduler.
Definition scheduler.h:140
bool is_worker() const
True if running on a worker thread, false otherwise.
Definition scheduler.h:673
virtual void local_wait_for_all(task &parent, task *child)=0
static generic_scheduler * create_worker(market &m, size_t index, bool geniune)
Initialize a scheduler for a worker thread.
void attach_mailbox(affinity_id id)
Definition scheduler.h:667
void nested_arena_entry(arena *, size_t)
Definition arena.cpp:729
void attach_arena(arena *, size_t index, bool is_master)
Definition arena.cpp:80
task * my_dummy_task
Fake root task created by slave threads.
Definition scheduler.h:186
market * my_market
The market I am in.
Definition scheduler.h:172
void free_task_pool()
Deallocate task pool that was allocated by means of allocate_task_pool.
A fast random number generator.
Definition tbb_misc.h:135

Copyright © 2005-2020 Intel Corporation. All Rights Reserved.

Intel, Pentium, Intel Xeon, Itanium, Intel XScale and VTune are registered trademarks or trademarks of Intel Corporation or its subsidiaries in the United States and other countries.

* Other names and brands may be claimed as the property of others.