Intel(R) Threading Building Blocks Doxygen Documentation version 4.2.3
Loading...
Searching...
No Matches
scheduler_common.h
Go to the documentation of this file.
1/*
2 Copyright (c) 2005-2020 Intel Corporation
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15*/
16
17#ifndef _TBB_scheduler_common_H
18#define _TBB_scheduler_common_H
19
20#include "tbb/tbb_machine.h"
22
23#include <string.h> // for memset, memcpy, memmove
24
25#include "tbb_statistics.h"
26
27#if TBB_USE_ASSERT > 1
28#include <stdio.h>
29#endif /* TBB_USE_ASSERT > 1 */
30
31/* Temporarily change "private" to "public" while including "tbb/task.h".
32 This hack allows us to avoid publishing internal types and methods
33 in the public header files just for sake of friend declarations. */
34#ifndef private
35 #define private public
36 #define undef_private
37#endif
38
39#include "tbb/task.h"
40#include "tbb/tbb_exception.h"
41
42#ifdef undef_private
43 #undef private
44#endif
45
46#ifndef __TBB_SCHEDULER_MUTEX_TYPE
47#define __TBB_SCHEDULER_MUTEX_TYPE tbb::spin_mutex
48#endif
49// TODO: add conditional inclusion based on specified type
50#include "tbb/spin_mutex.h"
51
52// This macro is an attempt to get rid of ugly ifdefs in the shared parts of the code.
53// It drops the second argument depending on whether the controlling macro is defined.
54// The first argument is just a convenience allowing to keep comma before the macro usage.
55#if __TBB_TASK_GROUP_CONTEXT
56 #define __TBB_CONTEXT_ARG1(context) context
57 #define __TBB_CONTEXT_ARG(arg1, context) arg1, context
58#else /* !__TBB_TASK_GROUP_CONTEXT */
59 #define __TBB_CONTEXT_ARG1(context)
60 #define __TBB_CONTEXT_ARG(arg1, context) arg1
61#endif /* !__TBB_TASK_GROUP_CONTEXT */
62
63#if __TBB_TASK_ISOLATION
64 #define __TBB_ISOLATION_EXPR(isolation) isolation
65 #define __TBB_ISOLATION_ARG(arg1, isolation) arg1, isolation
66#else
67 #define __TBB_ISOLATION_EXPR(isolation)
68 #define __TBB_ISOLATION_ARG(arg1, isolation) arg1
69#endif /* __TBB_TASK_ISOLATION */
70
71
72#if DO_TBB_TRACE
73#include <cstdio>
74#define TBB_TRACE(x) ((void)std::printf x)
75#else
76#define TBB_TRACE(x) ((void)(0))
77#endif /* DO_TBB_TRACE */
78
79#if !__TBB_CPU_CTL_ENV_PRESENT
80#include <fenv.h>
81#endif
82
83#if _MSC_VER && !defined(__INTEL_COMPILER)
84 // Workaround for overzealous compiler warnings
85 // These particular warnings are so ubiquitous that no attempt is made to narrow
86 // the scope of the warnings.
87 #pragma warning (disable: 4100 4127 4312 4244 4267 4706)
88#endif
89
90namespace tbb {
91namespace interface7 {
92namespace internal {
93class task_arena_base;
94class delegated_task;
95class wait_task;
96}}
97namespace internal {
98using namespace interface7::internal;
99
100class arena;
101template<typename SchedulerTraits> class custom_scheduler;
102class generic_scheduler;
103class governor;
104class mail_outbox;
105class market;
106class observer_proxy;
107class task_scheduler_observer_v3;
108
109#if __TBB_TASK_PRIORITY
110static const intptr_t num_priority_levels = 3;
111static const intptr_t normalized_normal_priority = (num_priority_levels - 1) / 2;
112
113inline intptr_t normalize_priority ( priority_t p ) {
114 return intptr_t(p - priority_low) / priority_stride_v4;
115}
116
117static const priority_t priority_from_normalized_rep[num_priority_levels] = {
119};
120
121inline void assert_priority_valid ( intptr_t p ) {
122 __TBB_ASSERT_EX( p >= 0 && p < num_priority_levels, NULL );
123}
124
125inline intptr_t& priority ( task& t ) {
126 return t.prefix().context->my_priority;
127}
128#else /* __TBB_TASK_PRIORITY */
129static const intptr_t num_priority_levels = 1;
130#endif /* __TBB_TASK_PRIORITY */
131
134
135#if __TBB_TASK_GROUP_CONTEXT
137
146extern uintptr_t the_context_state_propagation_epoch;
147
149
150typedef scheduler_mutex_type context_state_propagation_mutex_type;
151extern context_state_propagation_mutex_type the_context_state_propagation_mutex;
152#endif /* __TBB_TASK_GROUP_CONTEXT */
153
155const size_t task_alignment = 32;
156
158
160
167#if __TBB_PREVIEW_CRITICAL_TASKS
169 es_task_critical = 0x8,
170#endif
178 es_task_is_stolen = 0x80
180
181inline void reset_extra_state ( task *t ) {
183}
184
192
195
202
203//------------------------------------------------------------------------
204// Debugging support
205//------------------------------------------------------------------------
206
207#if TBB_USE_ASSERT
208
210
211template <typename T>
212void poison_value ( T& val ) { val = * punned_cast<T*>(&venom); }
213
215inline bool is_alive( uintptr_t v ) { return v != venom; }
216
219inline void assert_task_valid( const task* task ) {
220 __TBB_ASSERT( task!=NULL, NULL );
221 __TBB_ASSERT( !is_poisoned(&task), NULL );
222 __TBB_ASSERT( (uintptr_t)task % task_alignment == 0, "misaligned task" );
223 __TBB_ASSERT( task->prefix().ref_count >= 0, NULL);
224#if __TBB_RECYCLE_TO_ENQUEUE
225 __TBB_ASSERT( (unsigned)task->state()<=(unsigned)task::to_enqueue, "corrupt task (invalid state)" );
226#else
227 __TBB_ASSERT( (unsigned)task->state()<=(unsigned)task::recycle, "corrupt task (invalid state)" );
228#endif
229}
230
231#else /* !TBB_USE_ASSERT */
232
235#define poison_value(g) ((void)0)
236
237inline void assert_task_valid( const task* ) {}
238
239#endif /* !TBB_USE_ASSERT */
240
241//------------------------------------------------------------------------
242// Helpers
243//------------------------------------------------------------------------
244
245#if __TBB_TASK_GROUP_CONTEXT
246inline bool ConcurrentWaitsEnabled ( task& t ) {
248}
249
250inline bool CancellationInfoPresent ( task& t ) {
251 return t.prefix().context->my_cancellation_requested != 0;
252}
253
254#if TBB_USE_CAPTURED_EXCEPTION
255 inline tbb_exception* TbbCurrentException( task_group_context*, tbb_exception* src) { return src->move(); }
256 inline tbb_exception* TbbCurrentException( task_group_context* c, captured_exception* src) {
257 if( c->my_version_and_traits & task_group_context::exact_exception )
258 runtime_warning( "Exact exception propagation is requested by application but the linked library is built without support for it");
259 return src;
260 }
261 #define TbbRethrowException(TbbCapturedException) (TbbCapturedException)->throw_self()
262#else
263 // Using macro instead of an inline function here allows to avoid evaluation of the
264 // TbbCapturedException expression when exact propagation is enabled for the context.
265 #define TbbCurrentException(context, TbbCapturedException) \
266 context->my_version_and_traits & task_group_context::exact_exception \
267 ? tbb_exception_ptr::allocate() \
268 : tbb_exception_ptr::allocate( *(TbbCapturedException) );
269 #define TbbRethrowException(TbbCapturedException) \
270 { \
271 if( governor::rethrow_exception_broken() ) fix_broken_rethrow(); \
272 (TbbCapturedException)->throw_self(); \
273 }
274#endif /* !TBB_USE_CAPTURED_EXCEPTION */
275
276#define TbbRegisterCurrentException(context, TbbCapturedException) \
277 if ( context->cancel_group_execution() ) { \
278 /* We are the first to signal cancellation, so store the exception that caused it. */ \
279 context->my_exception = TbbCurrentException( context, TbbCapturedException ); \
280 }
281
282#define TbbCatchAll(context) \
283 catch ( tbb_exception& exc ) { \
284 TbbRegisterCurrentException( context, &exc ); \
285 } catch ( std::exception& exc ) { \
286 TbbRegisterCurrentException( context, captured_exception::allocate(typeid(exc).name(), exc.what()) ); \
287 } catch ( ... ) { \
288 TbbRegisterCurrentException( context, captured_exception::allocate("...", "Unidentified exception") );\
289 }
290
291#else /* !__TBB_TASK_GROUP_CONTEXT */
292
293inline bool ConcurrentWaitsEnabled ( task& t ) { return false; }
294
295#endif /* __TBB_TASK_GROUP_CONTEXT */
296
297inline void prolonged_pause() {
298#if defined(__TBB_time_stamp) && !__TBB_STEALING_PAUSE
299 // Assumption based on practice: 1000-2000 ticks seems to be a suitable invariant for the
300 // majority of platforms. Currently, skip platforms that define __TBB_STEALING_PAUSE
301 // because these platforms require very careful tuning.
303 const machine_tsc_t finish = prev + 1000;
304 atomic_backoff backoff;
305 do {
306 backoff.bounded_pause();
308 if ( curr <= prev )
309 // Possibly, the current logical thread is moved to another hardware thread or overflow is occurred.
310 break;
311 prev = curr;
312 } while ( prev < finish );
313#else
314#ifdef __TBB_STEALING_PAUSE
315 static const long PauseTime = __TBB_STEALING_PAUSE;
316#elif __TBB_ipf
317 static const long PauseTime = 1500;
318#else
319 static const long PauseTime = 80;
320#endif
321 // TODO IDEA: Update PauseTime adaptively?
322 __TBB_Pause(PauseTime);
323#endif
324}
325
326//------------------------------------------------------------------------
327// arena_slot
328//------------------------------------------------------------------------
330 //TODO: make this tbb:atomic<>.
332
334
335 // Synchronization of access to Task pool
340
342
344
345#if __TBB_PREVIEW_RESUMABLE_TASKS
347 tbb::atomic<bool>* my_scheduler_is_recalled;
348#endif
349};
350
353
354 unsigned hint_for_pop;
355
356#if __TBB_PREVIEW_CRITICAL_TASKS
358 unsigned hint_for_critical;
359#endif
360
362
364
367
370
371#if __TBB_STATISTICS
373 statistics_counters *my_counters;
374#endif /* __TBB_STATISTICS */
375};
376
377struct arena_slot : padded<arena_slot_line1>, padded<arena_slot_line2> {
378#if TBB_USE_ASSERT
379 void fill_with_canary_pattern ( size_t first, size_t last ) {
380 for ( size_t i = first; i < last; ++i )
381 poison_pointer(task_pool_ptr[i]);
382 }
383#else
384 void fill_with_canary_pattern ( size_t, size_t ) {}
385#endif /* TBB_USE_ASSERT */
386
387 void allocate_task_pool( size_t n ) {
388 size_t byte_size = ((n * sizeof(task*) + NFS_MaxLineSize - 1) / NFS_MaxLineSize) * NFS_MaxLineSize;
389 my_task_pool_size = byte_size / sizeof(task*);
390 task_pool_ptr = (task**)NFS_Allocate( 1, byte_size, NULL );
391 // No need to clear the fresh deque since valid items are designated by the head and tail members.
392 // But fill it with a canary pattern in the high vigilance debug mode.
393 fill_with_canary_pattern( 0, my_task_pool_size );
394 }
395
398 // TODO: understand the assertion and modify
399 // __TBB_ASSERT( !task_pool /*TODO: == EmptyTaskPool*/, NULL);
400 if( task_pool_ptr ) {
401 __TBB_ASSERT( my_task_pool_size, NULL);
402 NFS_Free( task_pool_ptr );
403 task_pool_ptr = NULL;
404 my_task_pool_size = 0;
405 }
406 }
407};
408
409#if !__TBB_CPU_CTL_ENV_PRESENT
410class cpu_ctl_env {
411 fenv_t *my_fenv_ptr;
412public:
415 if ( my_fenv_ptr )
417 }
418 // It is possible not to copy memory but just to copy pointers but the following issues should be addressed:
419 // 1. The arena lifetime and the context lifetime are independent;
420 // 2. The user is allowed to recapture different FPU settings to context so 'current FPU settings' inside
421 // dispatch loop may become invalid.
422 // But do we really want to improve the fenv implementation? It seems to be better to replace the fenv implementation
423 // with a platform specific implementation.
424 cpu_ctl_env( const cpu_ctl_env &src ) : my_fenv_ptr(NULL) {
425 *this = src;
426 }
428 __TBB_ASSERT( src.my_fenv_ptr, NULL );
429 if ( !my_fenv_ptr )
430 my_fenv_ptr = (fenv_t*)tbb::internal::NFS_Allocate(1, sizeof(fenv_t), NULL);
431 *my_fenv_ptr = *src.my_fenv_ptr;
432 return *this;
433 }
434 bool operator!=( const cpu_ctl_env &ctl ) const {
435 __TBB_ASSERT( my_fenv_ptr, "cpu_ctl_env is not initialized." );
436 __TBB_ASSERT( ctl.my_fenv_ptr, "cpu_ctl_env is not initialized." );
437 return memcmp( (void*)my_fenv_ptr, (void*)ctl.my_fenv_ptr, sizeof(fenv_t) );
438 }
439 void get_env () {
440 if ( !my_fenv_ptr )
441 my_fenv_ptr = (fenv_t*)tbb::internal::NFS_Allocate(1, sizeof(fenv_t), NULL);
442 fegetenv( my_fenv_ptr );
443 }
444 const cpu_ctl_env& set_env () const {
445 __TBB_ASSERT( my_fenv_ptr, "cpu_ctl_env is not initialized." );
446 fesetenv( my_fenv_ptr );
447 return *this;
448 }
449};
450#endif /* !__TBB_CPU_CTL_ENV_PRESENT */
451
452} // namespace internal
453} // namespace tbb
454
455#endif /* _TBB_scheduler_common_H */
#define __TBB_Pause(V)
Definition: gcc_arm.h:216
#define __TBB_time_stamp()
#define __TBB_STEALING_PAUSE
Definition: mic_common.h:40
#define __TBB_atomic
Definition: tbb_stddef.h:237
#define __TBB_ASSERT_EX(predicate, comment)
"Extended" version is useful to suppress warnings if a variable is only used with an assert
Definition: tbb_stddef.h:167
#define __TBB_ASSERT(predicate, comment)
No-op version of __TBB_ASSERT.
Definition: tbb_stddef.h:165
#define __TBB_SCHEDULER_MUTEX_TYPE
#define poison_value(g)
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task * task
void const char const char int ITT_FORMAT __itt_group_sync p
void *__TBB_EXPORTED_FUNC NFS_Allocate(size_t n_element, size_t element_size, void *hint)
Allocate memory on cache/sector line boundary.
const size_t NFS_MaxLineSize
Compile-time constant that is upper bound on cache line/sector size.
Definition: tbb_stddef.h:216
void __TBB_EXPORTED_FUNC NFS_Free(void *)
Free memory allocated by NFS_Allocate.
The graph class.
priority_t
Definition: task.h:317
@ priority_normal
Definition: task.h:318
@ priority_high
Definition: task.h:320
@ priority_low
Definition: task.h:319
const size_t task_prefix_reservation_size
Number of bytes reserved for a task prefix.
void __TBB_EXPORTED_FUNC runtime_warning(const char *format,...)
Report a runtime warning.
void assert_task_valid(const task *)
static const int priority_stride_v4
Definition: task.h:310
void poison_pointer(T *__TBB_atomic &)
Definition: tbb_stddef.h:305
task_extra_state
Definitions for bits in task_prefix::extra_state.
@ es_version_1_task
Tag for v1 tasks (i.e. tasks in TBB 1.0 and 2.0)
@ es_task_enqueued
Tag for enqueued tasks.
@ es_task_proxy
Tag for v3 task_proxy.
@ es_ref_count_active
Set if ref_count might be changed by another thread. Used for debugging.
@ es_task_is_stolen
Set if the task has been stolen.
@ es_version_3_task
Tag for v3 tasks (i.e. tasks in TBB 2.1-2.2)
__TBB_SCHEDULER_MUTEX_TYPE scheduler_mutex_type
Mutex type for global locks in the scheduler.
const size_t task_alignment
Alignment for a task object.
void reset_extra_state(task *t)
bool ConcurrentWaitsEnabled(task &t)
auto last(Container &c) -> decltype(begin(c))
auto first(Container &c) -> decltype(begin(c))
static const intptr_t num_priority_levels
free_task_hint
Optimization hint to free_task that enables it omit unnecessary tests and code.
@ no_cache
Disable caching for a small task.
@ small_task
Task is known to be a small task.
@ no_cache_small_task
Task is known to be a small task and must not be cached.
@ local_task
Task is known to have been allocated by this scheduler.
@ small_local_task
Bitwise-OR of local_task and small_task.
uint64_t machine_tsc_t
cpu_ctl_env(const cpu_ctl_env &src)
const cpu_ctl_env & set_env() const
cpu_ctl_env & operator=(const cpu_ctl_env &src)
bool operator!=(const cpu_ctl_env &ctl) const
Memory prefix to a task object.
Definition: task.h:203
unsigned char extra_state
Miscellaneous state that is not directly visible to users, stored as a byte for compactness.
Definition: task.h:292
task_group_context * context
Shared context that is used to communicate asynchronous state changes.
Definition: task.h:230
uintptr_t my_version_and_traits
Version for run-time checks and behavioral traits of the context.
Definition: task.h:446
Base class for user-defined tasks.
Definition: task.h:615
@ recycle
task to be recycled as continuation
Definition: task.h:647
internal::task_prefix & prefix(internal::version_tag *=NULL) const
Get reference to corresponding task_prefix.
Definition: task.h:1002
Class that implements exponential backoff.
Definition: tbb_machine.h:345
bool bounded_pause()
Pause for a few times and return false if saturated.
Definition: tbb_machine.h:372
Pads type T to fill out to a multiple of cache line size.
Definition: tbb_stddef.h:261
A template to select either 32-bit or 64-bit constant as compile time, depending on machine word size...
Definition: tbb_stddef.h:494
Work stealing task scheduler.
Definition: scheduler.h:140
__TBB_atomic size_t head
Index of the first ready task in the deque.
generic_scheduler * my_scheduler
Scheduler of the thread attached to the slot.
size_t my_task_pool_size
Capacity of the primary task pool (number of elements - pointers to task).
unsigned hint_for_pop
Hint provided for operations with the container of starvation-resistant tasks.
task **__TBB_atomic task_pool_ptr
Task pool of the scheduler that owns this slot.
__TBB_atomic size_t tail
Index of the element following the last ready task in the deque.
void allocate_task_pool(size_t n)
void fill_with_canary_pattern(size_t, size_t)
void free_task_pool()
Deallocate task pool that was allocated by means of allocate_task_pool.

Copyright © 2005-2020 Intel Corporation. All Rights Reserved.

Intel, Pentium, Intel Xeon, Itanium, Intel XScale and VTune are registered trademarks or trademarks of Intel Corporation or its subsidiaries in the United States and other countries.

* Other names and brands may be claimed as the property of others.