LLVM OpenMP* Runtime Library
z_Windows_NT_util.cpp
1 /*
2  * z_Windows_NT_util.cpp -- platform specific routines.
3  */
4 
5 //===----------------------------------------------------------------------===//
6 //
7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8 // See https://llvm.org/LICENSE.txt for license information.
9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "kmp.h"
14 #include "kmp_affinity.h"
15 #include "kmp_i18n.h"
16 #include "kmp_io.h"
17 #include "kmp_itt.h"
18 #include "kmp_wait_release.h"
19 
20 /* This code is related to NtQuerySystemInformation() function. This function
21  is used in the Load balance algorithm for OMP_DYNAMIC=true to find the
22  number of running threads in the system. */
23 
24 #include <ntsecapi.h> // UNICODE_STRING
25 #include <ntstatus.h>
26 
27 enum SYSTEM_INFORMATION_CLASS {
28  SystemProcessInformation = 5
29 }; // SYSTEM_INFORMATION_CLASS
30 
31 struct CLIENT_ID {
32  HANDLE UniqueProcess;
33  HANDLE UniqueThread;
34 }; // struct CLIENT_ID
35 
36 enum THREAD_STATE {
37  StateInitialized,
38  StateReady,
39  StateRunning,
40  StateStandby,
41  StateTerminated,
42  StateWait,
43  StateTransition,
44  StateUnknown
45 }; // enum THREAD_STATE
46 
47 struct VM_COUNTERS {
48  SIZE_T PeakVirtualSize;
49  SIZE_T VirtualSize;
50  ULONG PageFaultCount;
51  SIZE_T PeakWorkingSetSize;
52  SIZE_T WorkingSetSize;
53  SIZE_T QuotaPeakPagedPoolUsage;
54  SIZE_T QuotaPagedPoolUsage;
55  SIZE_T QuotaPeakNonPagedPoolUsage;
56  SIZE_T QuotaNonPagedPoolUsage;
57  SIZE_T PagefileUsage;
58  SIZE_T PeakPagefileUsage;
59  SIZE_T PrivatePageCount;
60 }; // struct VM_COUNTERS
61 
62 struct SYSTEM_THREAD {
63  LARGE_INTEGER KernelTime;
64  LARGE_INTEGER UserTime;
65  LARGE_INTEGER CreateTime;
66  ULONG WaitTime;
67  LPVOID StartAddress;
68  CLIENT_ID ClientId;
69  DWORD Priority;
70  LONG BasePriority;
71  ULONG ContextSwitchCount;
72  THREAD_STATE State;
73  ULONG WaitReason;
74 }; // SYSTEM_THREAD
75 
76 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, KernelTime) == 0);
77 #if KMP_ARCH_X86
78 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 28);
79 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 52);
80 #else
81 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 32);
82 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 68);
83 #endif
84 
85 struct SYSTEM_PROCESS_INFORMATION {
86  ULONG NextEntryOffset;
87  ULONG NumberOfThreads;
88  LARGE_INTEGER Reserved[3];
89  LARGE_INTEGER CreateTime;
90  LARGE_INTEGER UserTime;
91  LARGE_INTEGER KernelTime;
92  UNICODE_STRING ImageName;
93  DWORD BasePriority;
94  HANDLE ProcessId;
95  HANDLE ParentProcessId;
96  ULONG HandleCount;
97  ULONG Reserved2[2];
98  VM_COUNTERS VMCounters;
99  IO_COUNTERS IOCounters;
100  SYSTEM_THREAD Threads[1];
101 }; // SYSTEM_PROCESS_INFORMATION
102 typedef SYSTEM_PROCESS_INFORMATION *PSYSTEM_PROCESS_INFORMATION;
103 
104 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, NextEntryOffset) == 0);
105 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, CreateTime) == 32);
106 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ImageName) == 56);
107 #if KMP_ARCH_X86
108 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 68);
109 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 76);
110 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 88);
111 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 136);
112 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 184);
113 #else
114 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 80);
115 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 96);
116 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 112);
117 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 208);
118 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 256);
119 #endif
120 
121 typedef NTSTATUS(NTAPI *NtQuerySystemInformation_t)(SYSTEM_INFORMATION_CLASS,
122  PVOID, ULONG, PULONG);
123 NtQuerySystemInformation_t NtQuerySystemInformation = NULL;
124 
125 HMODULE ntdll = NULL;
126 
127 /* End of NtQuerySystemInformation()-related code */
128 
129 static HMODULE kernel32 = NULL;
130 
131 #if KMP_HANDLE_SIGNALS
132 typedef void (*sig_func_t)(int);
133 static sig_func_t __kmp_sighldrs[NSIG];
134 static int __kmp_siginstalled[NSIG];
135 #endif
136 
137 #if KMP_USE_MONITOR
138 static HANDLE __kmp_monitor_ev;
139 #endif
140 static kmp_int64 __kmp_win32_time;
141 double __kmp_win32_tick;
142 
143 int __kmp_init_runtime = FALSE;
144 CRITICAL_SECTION __kmp_win32_section;
145 
146 void __kmp_win32_mutex_init(kmp_win32_mutex_t *mx) {
147  InitializeCriticalSection(&mx->cs);
148 #if USE_ITT_BUILD
149  __kmp_itt_system_object_created(&mx->cs, "Critical Section");
150 #endif /* USE_ITT_BUILD */
151 }
152 
153 void __kmp_win32_mutex_destroy(kmp_win32_mutex_t *mx) {
154  DeleteCriticalSection(&mx->cs);
155 }
156 
157 void __kmp_win32_mutex_lock(kmp_win32_mutex_t *mx) {
158  EnterCriticalSection(&mx->cs);
159 }
160 
161 int __kmp_win32_mutex_trylock(kmp_win32_mutex_t *mx) {
162  return TryEnterCriticalSection(&mx->cs);
163 }
164 
165 void __kmp_win32_mutex_unlock(kmp_win32_mutex_t *mx) {
166  LeaveCriticalSection(&mx->cs);
167 }
168 
169 void __kmp_win32_cond_init(kmp_win32_cond_t *cv) {
170  cv->waiters_count_ = 0;
171  cv->wait_generation_count_ = 0;
172  cv->release_count_ = 0;
173 
174  /* Initialize the critical section */
175  __kmp_win32_mutex_init(&cv->waiters_count_lock_);
176 
177  /* Create a manual-reset event. */
178  cv->event_ = CreateEvent(NULL, // no security
179  TRUE, // manual-reset
180  FALSE, // non-signaled initially
181  NULL); // unnamed
182 #if USE_ITT_BUILD
183  __kmp_itt_system_object_created(cv->event_, "Event");
184 #endif /* USE_ITT_BUILD */
185 }
186 
187 void __kmp_win32_cond_destroy(kmp_win32_cond_t *cv) {
188  __kmp_win32_mutex_destroy(&cv->waiters_count_lock_);
189  __kmp_free_handle(cv->event_);
190  memset(cv, '\0', sizeof(*cv));
191 }
192 
193 /* TODO associate cv with a team instead of a thread so as to optimize
194  the case where we wake up a whole team */
195 
196 template <class C>
197 static void __kmp_win32_cond_wait(kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx,
198  kmp_info_t *th, C *flag) {
199  int my_generation;
200  int last_waiter;
201 
202  /* Avoid race conditions */
203  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
204 
205  /* Increment count of waiters */
206  cv->waiters_count_++;
207 
208  /* Store current generation in our activation record. */
209  my_generation = cv->wait_generation_count_;
210 
211  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
212  __kmp_win32_mutex_unlock(mx);
213 
214  for (;;) {
215  int wait_done = 0;
216  DWORD res, timeout = 5000; // just tried to quess an appropriate number
217  /* Wait until the event is signaled */
218  res = WaitForSingleObject(cv->event_, timeout);
219 
220  if (res == WAIT_OBJECT_0) {
221  // event signaled
222  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
223  /* Exit the loop when the <cv->event_> is signaled and there are still
224  waiting threads from this <wait_generation> that haven't been released
225  from this wait yet. */
226  wait_done = (cv->release_count_ > 0) &&
227  (cv->wait_generation_count_ != my_generation);
228  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
229  } else if (res == WAIT_TIMEOUT || res == WAIT_FAILED) {
230  // check if the flag and cv counters are in consistent state
231  // as MS sent us debug dump whith inconsistent state of data
232  __kmp_win32_mutex_lock(mx);
233  typename C::flag_t old_f = flag->set_sleeping();
234  if (!flag->done_check_val(old_f & ~KMP_BARRIER_SLEEP_STATE)) {
235  __kmp_win32_mutex_unlock(mx);
236  continue;
237  }
238  // condition fulfilled, exiting
239  old_f = flag->unset_sleeping();
240  KMP_DEBUG_ASSERT(old_f & KMP_BARRIER_SLEEP_STATE);
241  TCW_PTR(th->th.th_sleep_loc, NULL);
242  KF_TRACE(50,
243  ("__kmp_win32_cond_wait: exiting, condition "
244  "fulfilled: flag's loc(%p): %u => %u\n",
245  flag->get(), (unsigned int)old_f, (unsigned int)flag->load()));
246 
247  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
248  KMP_DEBUG_ASSERT(cv->waiters_count_ > 0);
249  cv->release_count_ = cv->waiters_count_;
250  cv->wait_generation_count_++;
251  wait_done = 1;
252  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
253 
254  __kmp_win32_mutex_unlock(mx);
255  }
256  /* there used to be a semicolon after the if statement, it looked like a
257  bug, so i removed it */
258  if (wait_done)
259  break;
260  }
261 
262  __kmp_win32_mutex_lock(mx);
263  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
264 
265  cv->waiters_count_--;
266  cv->release_count_--;
267 
268  last_waiter = (cv->release_count_ == 0);
269 
270  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
271 
272  if (last_waiter) {
273  /* We're the last waiter to be notified, so reset the manual event. */
274  ResetEvent(cv->event_);
275  }
276 }
277 
278 void __kmp_win32_cond_broadcast(kmp_win32_cond_t *cv) {
279  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
280 
281  if (cv->waiters_count_ > 0) {
282  SetEvent(cv->event_);
283  /* Release all the threads in this generation. */
284 
285  cv->release_count_ = cv->waiters_count_;
286 
287  /* Start a new generation. */
288  cv->wait_generation_count_++;
289  }
290 
291  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
292 }
293 
294 void __kmp_win32_cond_signal(kmp_win32_cond_t *cv) {
295  __kmp_win32_cond_broadcast(cv);
296 }
297 
298 void __kmp_enable(int new_state) {
299  if (__kmp_init_runtime)
300  LeaveCriticalSection(&__kmp_win32_section);
301 }
302 
303 void __kmp_disable(int *old_state) {
304  *old_state = 0;
305 
306  if (__kmp_init_runtime)
307  EnterCriticalSection(&__kmp_win32_section);
308 }
309 
310 void __kmp_suspend_initialize(void) { /* do nothing */
311 }
312 
313 void __kmp_suspend_initialize_thread(kmp_info_t *th) {
314  int old_value = KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init);
315  int new_value = TRUE;
316  // Return if already initialized
317  if (old_value == new_value)
318  return;
319  // Wait, then return if being initialized
320  if (old_value == -1 ||
321  !__kmp_atomic_compare_store(&th->th.th_suspend_init, old_value, -1)) {
322  while (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init) != new_value) {
323  KMP_CPU_PAUSE();
324  }
325  } else {
326  // Claim to be the initializer and do initializations
327  __kmp_win32_cond_init(&th->th.th_suspend_cv);
328  __kmp_win32_mutex_init(&th->th.th_suspend_mx);
329  KMP_ATOMIC_ST_REL(&th->th.th_suspend_init, new_value);
330  }
331 }
332 
333 void __kmp_suspend_uninitialize_thread(kmp_info_t *th) {
334  if (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init)) {
335  /* this means we have initialize the suspension pthread objects for this
336  thread in this instance of the process */
337  __kmp_win32_cond_destroy(&th->th.th_suspend_cv);
338  __kmp_win32_mutex_destroy(&th->th.th_suspend_mx);
339  KMP_ATOMIC_ST_REL(&th->th.th_suspend_init, FALSE);
340  }
341 }
342 
343 int __kmp_try_suspend_mx(kmp_info_t *th) {
344  return __kmp_win32_mutex_trylock(&th->th.th_suspend_mx);
345 }
346 
347 void __kmp_lock_suspend_mx(kmp_info_t *th) {
348  __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
349 }
350 
351 void __kmp_unlock_suspend_mx(kmp_info_t *th) {
352  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
353 }
354 
355 /* This routine puts the calling thread to sleep after setting the
356  sleep bit for the indicated flag variable to true. */
357 template <class C>
358 static inline void __kmp_suspend_template(int th_gtid, C *flag) {
359  kmp_info_t *th = __kmp_threads[th_gtid];
360  int status;
361  typename C::flag_t old_spin;
362 
363  KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag's loc(%p)\n",
364  th_gtid, flag->get()));
365 
366  __kmp_suspend_initialize_thread(th);
367  __kmp_lock_suspend_mx(th);
368 
369  KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for flag's"
370  " loc(%p)\n",
371  th_gtid, flag->get()));
372 
373  /* TODO: shouldn't this use release semantics to ensure that
374  __kmp_suspend_initialize_thread gets called first? */
375  old_spin = flag->set_sleeping();
376  if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
377  __kmp_pause_status != kmp_soft_paused) {
378  flag->unset_sleeping();
379  __kmp_unlock_suspend_mx(th);
380  return;
381  }
382 
383  KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for flag's"
384  " loc(%p)==%u\n",
385  th_gtid, flag->get(), (unsigned int)flag->load()));
386 
387  if (flag->done_check_val(old_spin)) {
388  old_spin = flag->unset_sleeping();
389  KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit "
390  "for flag's loc(%p)\n",
391  th_gtid, flag->get()));
392  } else {
393 #ifdef DEBUG_SUSPEND
394  __kmp_suspend_count++;
395 #endif
396  /* Encapsulate in a loop as the documentation states that this may "with
397  low probability" return when the condition variable has not been signaled
398  or broadcast */
399  int deactivated = FALSE;
400  TCW_PTR(th->th.th_sleep_loc, (void *)flag);
401  while (flag->is_sleeping()) {
402  KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform "
403  "kmp_win32_cond_wait()\n",
404  th_gtid));
405  // Mark the thread as no longer active (only in the first iteration of the
406  // loop).
407  if (!deactivated) {
408  th->th.th_active = FALSE;
409  if (th->th.th_active_in_pool) {
410  th->th.th_active_in_pool = FALSE;
411  KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
412  KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
413  }
414  deactivated = TRUE;
415  __kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, th,
416  flag);
417  } else {
418  __kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, th,
419  flag);
420  }
421 
422 #ifdef KMP_DEBUG
423  if (flag->is_sleeping()) {
424  KF_TRACE(100,
425  ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid));
426  }
427 #endif /* KMP_DEBUG */
428 
429  } // while
430 
431  // Mark the thread as active again (if it was previous marked as inactive)
432  if (deactivated) {
433  th->th.th_active = TRUE;
434  if (TCR_4(th->th.th_in_pool)) {
435  KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
436  th->th.th_active_in_pool = TRUE;
437  }
438  }
439  }
440 
441  __kmp_unlock_suspend_mx(th);
442  KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid));
443 }
444 
445 template <bool C, bool S>
446 void __kmp_suspend_32(int th_gtid, kmp_flag_32<C, S> *flag) {
447  __kmp_suspend_template(th_gtid, flag);
448 }
449 template <bool C, bool S>
450 void __kmp_suspend_64(int th_gtid, kmp_flag_64<C, S> *flag) {
451  __kmp_suspend_template(th_gtid, flag);
452 }
453 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
454  __kmp_suspend_template(th_gtid, flag);
455 }
456 
457 template void __kmp_suspend_32<false, false>(int, kmp_flag_32<false, false> *);
458 template void __kmp_suspend_64<false, true>(int, kmp_flag_64<false, true> *);
459 template void __kmp_suspend_64<true, false>(int, kmp_flag_64<true, false> *);
460 
461 /* This routine signals the thread specified by target_gtid to wake up
462  after setting the sleep bit indicated by the flag argument to FALSE */
463 template <class C>
464 static inline void __kmp_resume_template(int target_gtid, C *flag) {
465  kmp_info_t *th = __kmp_threads[target_gtid];
466  int status;
467 
468 #ifdef KMP_DEBUG
469  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
470 #endif
471 
472  KF_TRACE(30, ("__kmp_resume_template: T#%d wants to wakeup T#%d enter\n",
473  gtid, target_gtid));
474 
475  __kmp_suspend_initialize_thread(th);
476  __kmp_lock_suspend_mx(th);
477 
478  if (!flag) { // coming from __kmp_null_resume_wrapper
479  flag = (C *)th->th.th_sleep_loc;
480  }
481 
482  // First, check if the flag is null or its type has changed. If so, someone
483  // else woke it up.
484  if (!flag || flag->get_type() != flag->get_ptr_type()) { // get_ptr_type
485  // simply shows what
486  // flag was cast to
487  KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
488  "awake: flag's loc(%p)\n",
489  gtid, target_gtid, NULL));
490  __kmp_unlock_suspend_mx(th);
491  return;
492  } else {
493  typename C::flag_t old_spin = flag->unset_sleeping();
494  if (!flag->is_sleeping_val(old_spin)) {
495  KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
496  "awake: flag's loc(%p): %u => %u\n",
497  gtid, target_gtid, flag->get(), (unsigned int)old_spin,
498  (unsigned int)flag->load()));
499  __kmp_unlock_suspend_mx(th);
500  return;
501  }
502  }
503  TCW_PTR(th->th.th_sleep_loc, NULL);
504  KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep "
505  "bit for flag's loc(%p)\n",
506  gtid, target_gtid, flag->get()));
507 
508  __kmp_win32_cond_signal(&th->th.th_suspend_cv);
509  __kmp_unlock_suspend_mx(th);
510 
511  KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up"
512  " for T#%d\n",
513  gtid, target_gtid));
514 }
515 
516 template <bool C, bool S>
517 void __kmp_resume_32(int target_gtid, kmp_flag_32<C, S> *flag) {
518  __kmp_resume_template(target_gtid, flag);
519 }
520 template <bool C, bool S>
521 void __kmp_resume_64(int target_gtid, kmp_flag_64<C, S> *flag) {
522  __kmp_resume_template(target_gtid, flag);
523 }
524 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
525  __kmp_resume_template(target_gtid, flag);
526 }
527 
528 template void __kmp_resume_32<false, true>(int, kmp_flag_32<false, true> *);
529 template void __kmp_resume_64<false, true>(int, kmp_flag_64<false, true> *);
530 
531 void __kmp_yield() { Sleep(0); }
532 
533 void __kmp_gtid_set_specific(int gtid) {
534  if (__kmp_init_gtid) {
535  KA_TRACE(50, ("__kmp_gtid_set_specific: T#%d key:%d\n", gtid,
536  __kmp_gtid_threadprivate_key));
537  if (!TlsSetValue(__kmp_gtid_threadprivate_key, (LPVOID)(gtid + 1)))
538  KMP_FATAL(TLSSetValueFailed);
539  } else {
540  KA_TRACE(50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n"));
541  }
542 }
543 
544 int __kmp_gtid_get_specific() {
545  int gtid;
546  if (!__kmp_init_gtid) {
547  KA_TRACE(50, ("__kmp_gtid_get_specific: runtime shutdown, returning "
548  "KMP_GTID_SHUTDOWN\n"));
549  return KMP_GTID_SHUTDOWN;
550  }
551  gtid = (int)(kmp_intptr_t)TlsGetValue(__kmp_gtid_threadprivate_key);
552  if (gtid == 0) {
553  gtid = KMP_GTID_DNE;
554  } else {
555  gtid--;
556  }
557  KA_TRACE(50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
558  __kmp_gtid_threadprivate_key, gtid));
559  return gtid;
560 }
561 
562 void __kmp_affinity_bind_thread(int proc) {
563  if (__kmp_num_proc_groups > 1) {
564  // Form the GROUP_AFFINITY struct directly, rather than filling
565  // out a bit vector and calling __kmp_set_system_affinity().
566  GROUP_AFFINITY ga;
567  KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups * CHAR_BIT *
568  sizeof(DWORD_PTR))));
569  ga.Group = proc / (CHAR_BIT * sizeof(DWORD_PTR));
570  ga.Mask = (unsigned long long)1 << (proc % (CHAR_BIT * sizeof(DWORD_PTR)));
571  ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
572 
573  KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
574  if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
575  DWORD error = GetLastError();
576  if (__kmp_affinity_verbose) { // AC: continue silently if not verbose
577  kmp_msg_t err_code = KMP_ERR(error);
578  __kmp_msg(kmp_ms_warning, KMP_MSG(CantSetThreadAffMask), err_code,
579  __kmp_msg_null);
580  if (__kmp_generate_warnings == kmp_warnings_off) {
581  __kmp_str_free(&err_code.str);
582  }
583  }
584  }
585  } else {
586  kmp_affin_mask_t *mask;
587  KMP_CPU_ALLOC_ON_STACK(mask);
588  KMP_CPU_ZERO(mask);
589  KMP_CPU_SET(proc, mask);
590  __kmp_set_system_affinity(mask, TRUE);
591  KMP_CPU_FREE_FROM_STACK(mask);
592  }
593 }
594 
595 void __kmp_affinity_determine_capable(const char *env_var) {
596  // All versions of Windows* OS (since Win '95) support
597  // SetThreadAffinityMask().
598 
599 #if KMP_GROUP_AFFINITY
600  KMP_AFFINITY_ENABLE(__kmp_num_proc_groups * sizeof(DWORD_PTR));
601 #else
602  KMP_AFFINITY_ENABLE(sizeof(DWORD_PTR));
603 #endif
604 
605  KA_TRACE(10, ("__kmp_affinity_determine_capable: "
606  "Windows* OS affinity interface functional (mask size = "
607  "%" KMP_SIZE_T_SPEC ").\n",
608  __kmp_affin_mask_size));
609 }
610 
611 double __kmp_read_cpu_time(void) {
612  FILETIME CreationTime, ExitTime, KernelTime, UserTime;
613  int status;
614  double cpu_time;
615 
616  cpu_time = 0;
617 
618  status = GetProcessTimes(GetCurrentProcess(), &CreationTime, &ExitTime,
619  &KernelTime, &UserTime);
620 
621  if (status) {
622  double sec = 0;
623 
624  sec += KernelTime.dwHighDateTime;
625  sec += UserTime.dwHighDateTime;
626 
627  /* Shift left by 32 bits */
628  sec *= (double)(1 << 16) * (double)(1 << 16);
629 
630  sec += KernelTime.dwLowDateTime;
631  sec += UserTime.dwLowDateTime;
632 
633  cpu_time += (sec * 100.0) / KMP_NSEC_PER_SEC;
634  }
635 
636  return cpu_time;
637 }
638 
639 int __kmp_read_system_info(struct kmp_sys_info *info) {
640  info->maxrss = 0; /* the maximum resident set size utilized (in kilobytes) */
641  info->minflt = 0; /* the number of page faults serviced without any I/O */
642  info->majflt = 0; /* the number of page faults serviced that required I/O */
643  info->nswap = 0; // the number of times a process was "swapped" out of memory
644  info->inblock = 0; // the number of times the file system had to perform input
645  info->oublock = 0; // number of times the file system had to perform output
646  info->nvcsw = 0; /* the number of times a context switch was voluntarily */
647  info->nivcsw = 0; /* the number of times a context switch was forced */
648 
649  return 1;
650 }
651 
652 void __kmp_runtime_initialize(void) {
653  SYSTEM_INFO info;
654  kmp_str_buf_t path;
655  UINT path_size;
656 
657  if (__kmp_init_runtime) {
658  return;
659  }
660 
661 #if KMP_DYNAMIC_LIB
662  /* Pin dynamic library for the lifetime of application */
663  {
664  // First, turn off error message boxes
665  UINT err_mode = SetErrorMode(SEM_FAILCRITICALERRORS);
666  HMODULE h;
667  BOOL ret = GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
668  GET_MODULE_HANDLE_EX_FLAG_PIN,
669  (LPCTSTR)&__kmp_serial_initialize, &h);
670  KMP_DEBUG_ASSERT2(h && ret, "OpenMP RTL cannot find itself loaded");
671  SetErrorMode(err_mode); // Restore error mode
672  KA_TRACE(10, ("__kmp_runtime_initialize: dynamic library pinned\n"));
673  }
674 #endif
675 
676  InitializeCriticalSection(&__kmp_win32_section);
677 #if USE_ITT_BUILD
678  __kmp_itt_system_object_created(&__kmp_win32_section, "Critical Section");
679 #endif /* USE_ITT_BUILD */
680  __kmp_initialize_system_tick();
681 
682 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
683  if (!__kmp_cpuinfo.initialized) {
684  __kmp_query_cpuid(&__kmp_cpuinfo);
685  }
686 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
687 
688 /* Set up minimum number of threads to switch to TLS gtid */
689 #if KMP_OS_WINDOWS && !KMP_DYNAMIC_LIB
690  // Windows* OS, static library.
691  /* New thread may use stack space previously used by another thread,
692  currently terminated. On Windows* OS, in case of static linking, we do not
693  know the moment of thread termination, and our structures (__kmp_threads
694  and __kmp_root arrays) are still keep info about dead threads. This leads
695  to problem in __kmp_get_global_thread_id() function: it wrongly finds gtid
696  (by searching through stack addresses of all known threads) for
697  unregistered foreign tread.
698 
699  Setting __kmp_tls_gtid_min to 0 workarounds this problem:
700  __kmp_get_global_thread_id() does not search through stacks, but get gtid
701  from TLS immediately.
702  --ln
703  */
704  __kmp_tls_gtid_min = 0;
705 #else
706  __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
707 #endif
708 
709  /* for the static library */
710  if (!__kmp_gtid_threadprivate_key) {
711  __kmp_gtid_threadprivate_key = TlsAlloc();
712  if (__kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES) {
713  KMP_FATAL(TLSOutOfIndexes);
714  }
715  }
716 
717  // Load ntdll.dll.
718  /* Simple GetModuleHandle( "ntdll.dl" ) is not suitable due to security issue
719  (see http://www.microsoft.com/technet/security/advisory/2269637.mspx). We
720  have to specify full path to the library. */
721  __kmp_str_buf_init(&path);
722  path_size = GetSystemDirectory(path.str, path.size);
723  KMP_DEBUG_ASSERT(path_size > 0);
724  if (path_size >= path.size) {
725  // Buffer is too short. Expand the buffer and try again.
726  __kmp_str_buf_reserve(&path, path_size);
727  path_size = GetSystemDirectory(path.str, path.size);
728  KMP_DEBUG_ASSERT(path_size > 0);
729  }
730  if (path_size > 0 && path_size < path.size) {
731  // Now we have system directory name in the buffer.
732  // Append backslash and name of dll to form full path,
733  path.used = path_size;
734  __kmp_str_buf_print(&path, "\\%s", "ntdll.dll");
735 
736  // Now load ntdll using full path.
737  ntdll = GetModuleHandle(path.str);
738  }
739 
740  KMP_DEBUG_ASSERT(ntdll != NULL);
741  if (ntdll != NULL) {
742  NtQuerySystemInformation = (NtQuerySystemInformation_t)GetProcAddress(
743  ntdll, "NtQuerySystemInformation");
744  }
745  KMP_DEBUG_ASSERT(NtQuerySystemInformation != NULL);
746 
747 #if KMP_GROUP_AFFINITY
748  // Load kernel32.dll.
749  // Same caveat - must use full system path name.
750  if (path_size > 0 && path_size < path.size) {
751  // Truncate the buffer back to just the system path length,
752  // discarding "\\ntdll.dll", and replacing it with "kernel32.dll".
753  path.used = path_size;
754  __kmp_str_buf_print(&path, "\\%s", "kernel32.dll");
755 
756  // Load kernel32.dll using full path.
757  kernel32 = GetModuleHandle(path.str);
758  KA_TRACE(10, ("__kmp_runtime_initialize: kernel32.dll = %s\n", path.str));
759 
760  // Load the function pointers to kernel32.dll routines
761  // that may or may not exist on this system.
762  if (kernel32 != NULL) {
763  __kmp_GetActiveProcessorCount =
764  (kmp_GetActiveProcessorCount_t)GetProcAddress(
765  kernel32, "GetActiveProcessorCount");
766  __kmp_GetActiveProcessorGroupCount =
767  (kmp_GetActiveProcessorGroupCount_t)GetProcAddress(
768  kernel32, "GetActiveProcessorGroupCount");
769  __kmp_GetThreadGroupAffinity =
770  (kmp_GetThreadGroupAffinity_t)GetProcAddress(
771  kernel32, "GetThreadGroupAffinity");
772  __kmp_SetThreadGroupAffinity =
773  (kmp_SetThreadGroupAffinity_t)GetProcAddress(
774  kernel32, "SetThreadGroupAffinity");
775 
776  KA_TRACE(10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorCount"
777  " = %p\n",
778  __kmp_GetActiveProcessorCount));
779  KA_TRACE(10, ("__kmp_runtime_initialize: "
780  "__kmp_GetActiveProcessorGroupCount = %p\n",
781  __kmp_GetActiveProcessorGroupCount));
782  KA_TRACE(10, ("__kmp_runtime_initialize:__kmp_GetThreadGroupAffinity"
783  " = %p\n",
784  __kmp_GetThreadGroupAffinity));
785  KA_TRACE(10, ("__kmp_runtime_initialize: __kmp_SetThreadGroupAffinity"
786  " = %p\n",
787  __kmp_SetThreadGroupAffinity));
788  KA_TRACE(10, ("__kmp_runtime_initialize: sizeof(kmp_affin_mask_t) = %d\n",
789  sizeof(kmp_affin_mask_t)));
790 
791  // See if group affinity is supported on this system.
792  // If so, calculate the #groups and #procs.
793  //
794  // Group affinity was introduced with Windows* 7 OS and
795  // Windows* Server 2008 R2 OS.
796  if ((__kmp_GetActiveProcessorCount != NULL) &&
797  (__kmp_GetActiveProcessorGroupCount != NULL) &&
798  (__kmp_GetThreadGroupAffinity != NULL) &&
799  (__kmp_SetThreadGroupAffinity != NULL) &&
800  ((__kmp_num_proc_groups = __kmp_GetActiveProcessorGroupCount()) >
801  1)) {
802  // Calculate the total number of active OS procs.
803  int i;
804 
805  KA_TRACE(10, ("__kmp_runtime_initialize: %d processor groups"
806  " detected\n",
807  __kmp_num_proc_groups));
808 
809  __kmp_xproc = 0;
810 
811  for (i = 0; i < __kmp_num_proc_groups; i++) {
812  DWORD size = __kmp_GetActiveProcessorCount(i);
813  __kmp_xproc += size;
814  KA_TRACE(10, ("__kmp_runtime_initialize: proc group %d size = %d\n",
815  i, size));
816  }
817  } else {
818  KA_TRACE(10, ("__kmp_runtime_initialize: %d processor groups"
819  " detected\n",
820  __kmp_num_proc_groups));
821  }
822  }
823  }
824  if (__kmp_num_proc_groups <= 1) {
825  GetSystemInfo(&info);
826  __kmp_xproc = info.dwNumberOfProcessors;
827  }
828 #else
829  GetSystemInfo(&info);
830  __kmp_xproc = info.dwNumberOfProcessors;
831 #endif /* KMP_GROUP_AFFINITY */
832 
833  // If the OS said there were 0 procs, take a guess and use a value of 2.
834  // This is done for Linux* OS, also. Do we need error / warning?
835  if (__kmp_xproc <= 0) {
836  __kmp_xproc = 2;
837  }
838 
839  KA_TRACE(5,
840  ("__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc));
841 
842  __kmp_str_buf_free(&path);
843 
844 #if USE_ITT_BUILD
845  __kmp_itt_initialize();
846 #endif /* USE_ITT_BUILD */
847 
848  __kmp_init_runtime = TRUE;
849 } // __kmp_runtime_initialize
850 
851 void __kmp_runtime_destroy(void) {
852  if (!__kmp_init_runtime) {
853  return;
854  }
855 
856 #if USE_ITT_BUILD
857  __kmp_itt_destroy();
858 #endif /* USE_ITT_BUILD */
859 
860  /* we can't DeleteCriticalsection( & __kmp_win32_section ); */
861  /* due to the KX_TRACE() commands */
862  KA_TRACE(40, ("__kmp_runtime_destroy\n"));
863 
864  if (__kmp_gtid_threadprivate_key) {
865  TlsFree(__kmp_gtid_threadprivate_key);
866  __kmp_gtid_threadprivate_key = 0;
867  }
868 
869  __kmp_affinity_uninitialize();
870  DeleteCriticalSection(&__kmp_win32_section);
871 
872  ntdll = NULL;
873  NtQuerySystemInformation = NULL;
874 
875 #if KMP_ARCH_X86_64
876  kernel32 = NULL;
877  __kmp_GetActiveProcessorCount = NULL;
878  __kmp_GetActiveProcessorGroupCount = NULL;
879  __kmp_GetThreadGroupAffinity = NULL;
880  __kmp_SetThreadGroupAffinity = NULL;
881 #endif // KMP_ARCH_X86_64
882 
883  __kmp_init_runtime = FALSE;
884 }
885 
886 void __kmp_terminate_thread(int gtid) {
887  kmp_info_t *th = __kmp_threads[gtid];
888 
889  if (!th)
890  return;
891 
892  KA_TRACE(10, ("__kmp_terminate_thread: kill (%d)\n", gtid));
893 
894  if (TerminateThread(th->th.th_info.ds.ds_thread, (DWORD)-1) == FALSE) {
895  /* It's OK, the thread may have exited already */
896  }
897  __kmp_free_handle(th->th.th_info.ds.ds_thread);
898 }
899 
900 void __kmp_clear_system_time(void) {
901  BOOL status;
902  LARGE_INTEGER time;
903  status = QueryPerformanceCounter(&time);
904  __kmp_win32_time = (kmp_int64)time.QuadPart;
905 }
906 
907 void __kmp_initialize_system_tick(void) {
908  {
909  BOOL status;
910  LARGE_INTEGER freq;
911 
912  status = QueryPerformanceFrequency(&freq);
913  if (!status) {
914  DWORD error = GetLastError();
915  __kmp_fatal(KMP_MSG(FunctionError, "QueryPerformanceFrequency()"),
916  KMP_ERR(error), __kmp_msg_null);
917 
918  } else {
919  __kmp_win32_tick = ((double)1.0) / (double)freq.QuadPart;
920  }
921  }
922 }
923 
924 /* Calculate the elapsed wall clock time for the user */
925 
926 void __kmp_elapsed(double *t) {
927  BOOL status;
928  LARGE_INTEGER now;
929  status = QueryPerformanceCounter(&now);
930  *t = ((double)now.QuadPart) * __kmp_win32_tick;
931 }
932 
933 /* Calculate the elapsed wall clock tick for the user */
934 
935 void __kmp_elapsed_tick(double *t) { *t = __kmp_win32_tick; }
936 
937 void __kmp_read_system_time(double *delta) {
938  if (delta != NULL) {
939  BOOL status;
940  LARGE_INTEGER now;
941 
942  status = QueryPerformanceCounter(&now);
943 
944  *delta = ((double)(((kmp_int64)now.QuadPart) - __kmp_win32_time)) *
945  __kmp_win32_tick;
946  }
947 }
948 
949 /* Return the current time stamp in nsec */
950 kmp_uint64 __kmp_now_nsec() {
951  LARGE_INTEGER now;
952  QueryPerformanceCounter(&now);
953  return 1e9 * __kmp_win32_tick * now.QuadPart;
954 }
955 
956 extern "C" void *__stdcall __kmp_launch_worker(void *arg) {
957  volatile void *stack_data;
958  void *exit_val;
959  void *padding = 0;
960  kmp_info_t *this_thr = (kmp_info_t *)arg;
961  int gtid;
962 
963  gtid = this_thr->th.th_info.ds.ds_gtid;
964  __kmp_gtid_set_specific(gtid);
965 #ifdef KMP_TDATA_GTID
966 #error "This define causes problems with LoadLibrary() + declspec(thread) " \
967  "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
968  "reference: http://support.microsoft.com/kb/118816"
969 //__kmp_gtid = gtid;
970 #endif
971 
972 #if USE_ITT_BUILD
973  __kmp_itt_thread_name(gtid);
974 #endif /* USE_ITT_BUILD */
975 
976  __kmp_affinity_set_init_mask(gtid, FALSE);
977 
978 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
979  // Set FP control regs to be a copy of the parallel initialization thread's.
980  __kmp_clear_x87_fpu_status_word();
981  __kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
982  __kmp_load_mxcsr(&__kmp_init_mxcsr);
983 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
984 
985  if (__kmp_stkoffset > 0 && gtid > 0) {
986  padding = KMP_ALLOCA(gtid * __kmp_stkoffset);
987  }
988 
989  KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
990  this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
991  TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
992 
993  if (TCR_4(__kmp_gtid_mode) <
994  2) { // check stack only if it is used to get gtid
995  TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data);
996  KMP_ASSERT(this_thr->th.th_info.ds.ds_stackgrow == FALSE);
997  __kmp_check_stack_overlap(this_thr);
998  }
999  KMP_MB();
1000  exit_val = __kmp_launch_thread(this_thr);
1001  KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
1002  TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
1003  KMP_MB();
1004  return exit_val;
1005 }
1006 
1007 #if KMP_USE_MONITOR
1008 /* The monitor thread controls all of the threads in the complex */
1009 
1010 void *__stdcall __kmp_launch_monitor(void *arg) {
1011  DWORD wait_status;
1012  kmp_thread_t monitor;
1013  int status;
1014  int interval;
1015  kmp_info_t *this_thr = (kmp_info_t *)arg;
1016 
1017  KMP_DEBUG_ASSERT(__kmp_init_monitor);
1018  TCW_4(__kmp_init_monitor, 2); // AC: Signal library that monitor has started
1019  // TODO: hide "2" in enum (like {true,false,started})
1020  this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1021  TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
1022 
1023  KMP_MB(); /* Flush all pending memory write invalidates. */
1024  KA_TRACE(10, ("__kmp_launch_monitor: launched\n"));
1025 
1026  monitor = GetCurrentThread();
1027 
1028  /* set thread priority */
1029  status = SetThreadPriority(monitor, THREAD_PRIORITY_HIGHEST);
1030  if (!status) {
1031  DWORD error = GetLastError();
1032  __kmp_fatal(KMP_MSG(CantSetThreadPriority), KMP_ERR(error), __kmp_msg_null);
1033  }
1034 
1035  /* register us as monitor */
1036  __kmp_gtid_set_specific(KMP_GTID_MONITOR);
1037 #ifdef KMP_TDATA_GTID
1038 #error "This define causes problems with LoadLibrary() + declspec(thread) " \
1039  "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
1040  "reference: http://support.microsoft.com/kb/118816"
1041 //__kmp_gtid = KMP_GTID_MONITOR;
1042 #endif
1043 
1044 #if USE_ITT_BUILD
1045  __kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore
1046 // monitor thread.
1047 #endif /* USE_ITT_BUILD */
1048 
1049  KMP_MB(); /* Flush all pending memory write invalidates. */
1050 
1051  interval = (1000 / __kmp_monitor_wakeups); /* in milliseconds */
1052 
1053  while (!TCR_4(__kmp_global.g.g_done)) {
1054  /* This thread monitors the state of the system */
1055 
1056  KA_TRACE(15, ("__kmp_launch_monitor: update\n"));
1057 
1058  wait_status = WaitForSingleObject(__kmp_monitor_ev, interval);
1059 
1060  if (wait_status == WAIT_TIMEOUT) {
1061  TCW_4(__kmp_global.g.g_time.dt.t_value,
1062  TCR_4(__kmp_global.g.g_time.dt.t_value) + 1);
1063  }
1064 
1065  KMP_MB(); /* Flush all pending memory write invalidates. */
1066  }
1067 
1068  KA_TRACE(10, ("__kmp_launch_monitor: finished\n"));
1069 
1070  status = SetThreadPriority(monitor, THREAD_PRIORITY_NORMAL);
1071  if (!status) {
1072  DWORD error = GetLastError();
1073  __kmp_fatal(KMP_MSG(CantSetThreadPriority), KMP_ERR(error), __kmp_msg_null);
1074  }
1075 
1076  if (__kmp_global.g.g_abort != 0) {
1077  /* now we need to terminate the worker threads */
1078  /* the value of t_abort is the signal we caught */
1079  int gtid;
1080 
1081  KA_TRACE(10, ("__kmp_launch_monitor: terminate sig=%d\n",
1082  (__kmp_global.g.g_abort)));
1083 
1084  /* terminate the OpenMP worker threads */
1085  /* TODO this is not valid for sibling threads!!
1086  * the uber master might not be 0 anymore.. */
1087  for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
1088  __kmp_terminate_thread(gtid);
1089 
1090  __kmp_cleanup();
1091 
1092  Sleep(0);
1093 
1094  KA_TRACE(10,
1095  ("__kmp_launch_monitor: raise sig=%d\n", __kmp_global.g.g_abort));
1096 
1097  if (__kmp_global.g.g_abort > 0) {
1098  raise(__kmp_global.g.g_abort);
1099  }
1100  }
1101 
1102  TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
1103 
1104  KMP_MB();
1105  return arg;
1106 }
1107 #endif
1108 
1109 void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size) {
1110  kmp_thread_t handle;
1111  DWORD idThread;
1112 
1113  KA_TRACE(10, ("__kmp_create_worker: try to create thread (%d)\n", gtid));
1114 
1115  th->th.th_info.ds.ds_gtid = gtid;
1116 
1117  if (KMP_UBER_GTID(gtid)) {
1118  int stack_data;
1119 
1120  /* TODO: GetCurrentThread() returns a pseudo-handle that is unsuitable for
1121  other threads to use. Is it appropriate to just use GetCurrentThread?
1122  When should we close this handle? When unregistering the root? */
1123  {
1124  BOOL rc;
1125  rc = DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
1126  GetCurrentProcess(), &th->th.th_info.ds.ds_thread, 0,
1127  FALSE, DUPLICATE_SAME_ACCESS);
1128  KMP_ASSERT(rc);
1129  KA_TRACE(10, (" __kmp_create_worker: ROOT Handle duplicated, th = %p, "
1130  "handle = %" KMP_UINTPTR_SPEC "\n",
1131  (LPVOID)th, th->th.th_info.ds.ds_thread));
1132  th->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1133  }
1134  if (TCR_4(__kmp_gtid_mode) < 2) { // check stack only if used to get gtid
1135  /* we will dynamically update the stack range if gtid_mode == 1 */
1136  TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
1137  TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
1138  TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
1139  __kmp_check_stack_overlap(th);
1140  }
1141  } else {
1142  KMP_MB(); /* Flush all pending memory write invalidates. */
1143 
1144  /* Set stack size for this thread now. */
1145  KA_TRACE(10,
1146  ("__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC " bytes\n",
1147  stack_size));
1148 
1149  stack_size += gtid * __kmp_stkoffset;
1150 
1151  TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size);
1152  TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
1153 
1154  KA_TRACE(10,
1155  ("__kmp_create_worker: (before) stack_size = %" KMP_SIZE_T_SPEC
1156  " bytes, &__kmp_launch_worker = %p, th = %p, &idThread = %p\n",
1157  (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
1158  (LPVOID)th, &idThread));
1159 
1160  handle = CreateThread(
1161  NULL, (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)__kmp_launch_worker,
1162  (LPVOID)th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
1163 
1164  KA_TRACE(10,
1165  ("__kmp_create_worker: (after) stack_size = %" KMP_SIZE_T_SPEC
1166  " bytes, &__kmp_launch_worker = %p, th = %p, "
1167  "idThread = %u, handle = %" KMP_UINTPTR_SPEC "\n",
1168  (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
1169  (LPVOID)th, idThread, handle));
1170 
1171  if (handle == 0) {
1172  DWORD error = GetLastError();
1173  __kmp_fatal(KMP_MSG(CantCreateThread), KMP_ERR(error), __kmp_msg_null);
1174  } else {
1175  th->th.th_info.ds.ds_thread = handle;
1176  }
1177 
1178  KMP_MB(); /* Flush all pending memory write invalidates. */
1179  }
1180 
1181  KA_TRACE(10, ("__kmp_create_worker: done creating thread (%d)\n", gtid));
1182 }
1183 
1184 int __kmp_still_running(kmp_info_t *th) {
1185  return (WAIT_TIMEOUT == WaitForSingleObject(th->th.th_info.ds.ds_thread, 0));
1186 }
1187 
1188 #if KMP_USE_MONITOR
1189 void __kmp_create_monitor(kmp_info_t *th) {
1190  kmp_thread_t handle;
1191  DWORD idThread;
1192  int ideal, new_ideal;
1193 
1194  if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
1195  // We don't need monitor thread in case of MAX_BLOCKTIME
1196  KA_TRACE(10, ("__kmp_create_monitor: skipping monitor thread because of "
1197  "MAX blocktime\n"));
1198  th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op
1199  th->th.th_info.ds.ds_gtid = 0;
1200  TCW_4(__kmp_init_monitor, 2); // Signal to stop waiting for monitor creation
1201  return;
1202  }
1203  KA_TRACE(10, ("__kmp_create_monitor: try to create monitor\n"));
1204 
1205  KMP_MB(); /* Flush all pending memory write invalidates. */
1206 
1207  __kmp_monitor_ev = CreateEvent(NULL, TRUE, FALSE, NULL);
1208  if (__kmp_monitor_ev == NULL) {
1209  DWORD error = GetLastError();
1210  __kmp_fatal(KMP_MSG(CantCreateEvent), KMP_ERR(error), __kmp_msg_null);
1211  }
1212 #if USE_ITT_BUILD
1213  __kmp_itt_system_object_created(__kmp_monitor_ev, "Event");
1214 #endif /* USE_ITT_BUILD */
1215 
1216  th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1217  th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1218 
1219  // FIXME - on Windows* OS, if __kmp_monitor_stksize = 0, figure out how
1220  // to automatically expand stacksize based on CreateThread error code.
1221  if (__kmp_monitor_stksize == 0) {
1222  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1223  }
1224  if (__kmp_monitor_stksize < __kmp_sys_min_stksize) {
1225  __kmp_monitor_stksize = __kmp_sys_min_stksize;
1226  }
1227 
1228  KA_TRACE(10, ("__kmp_create_monitor: requested stacksize = %d bytes\n",
1229  (int)__kmp_monitor_stksize));
1230 
1231  TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
1232 
1233  handle =
1234  CreateThread(NULL, (SIZE_T)__kmp_monitor_stksize,
1235  (LPTHREAD_START_ROUTINE)__kmp_launch_monitor, (LPVOID)th,
1236  STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
1237  if (handle == 0) {
1238  DWORD error = GetLastError();
1239  __kmp_fatal(KMP_MSG(CantCreateThread), KMP_ERR(error), __kmp_msg_null);
1240  } else
1241  th->th.th_info.ds.ds_thread = handle;
1242 
1243  KMP_MB(); /* Flush all pending memory write invalidates. */
1244 
1245  KA_TRACE(10, ("__kmp_create_monitor: monitor created %p\n",
1246  (void *)th->th.th_info.ds.ds_thread));
1247 }
1248 #endif
1249 
1250 /* Check to see if thread is still alive.
1251  NOTE: The ExitProcess(code) system call causes all threads to Terminate
1252  with a exit_val = code. Because of this we can not rely on exit_val having
1253  any particular value. So this routine may return STILL_ALIVE in exit_val
1254  even after the thread is dead. */
1255 
1256 int __kmp_is_thread_alive(kmp_info_t *th, DWORD *exit_val) {
1257  DWORD rc;
1258  rc = GetExitCodeThread(th->th.th_info.ds.ds_thread, exit_val);
1259  if (rc == 0) {
1260  DWORD error = GetLastError();
1261  __kmp_fatal(KMP_MSG(FunctionError, "GetExitCodeThread()"), KMP_ERR(error),
1262  __kmp_msg_null);
1263  }
1264  return (*exit_val == STILL_ACTIVE);
1265 }
1266 
1267 void __kmp_exit_thread(int exit_status) {
1268  ExitThread(exit_status);
1269 } // __kmp_exit_thread
1270 
1271 // This is a common part for both __kmp_reap_worker() and __kmp_reap_monitor().
1272 static void __kmp_reap_common(kmp_info_t *th) {
1273  DWORD exit_val;
1274 
1275  KMP_MB(); /* Flush all pending memory write invalidates. */
1276 
1277  KA_TRACE(
1278  10, ("__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid));
1279 
1280  /* 2006-10-19:
1281  There are two opposite situations:
1282  1. Windows* OS keep thread alive after it resets ds_alive flag and
1283  exits from thread function. (For example, see C70770/Q394281 "unloading of
1284  dll based on OMP is very slow".)
1285  2. Windows* OS may kill thread before it resets ds_alive flag.
1286 
1287  Right solution seems to be waiting for *either* thread termination *or*
1288  ds_alive resetting. */
1289  {
1290  // TODO: This code is very similar to KMP_WAIT. Need to generalize
1291  // KMP_WAIT to cover this usage also.
1292  void *obj = NULL;
1293  kmp_uint32 spins;
1294 #if USE_ITT_BUILD
1295  KMP_FSYNC_SPIN_INIT(obj, (void *)&th->th.th_info.ds.ds_alive);
1296 #endif /* USE_ITT_BUILD */
1297  KMP_INIT_YIELD(spins);
1298  do {
1299 #if USE_ITT_BUILD
1300  KMP_FSYNC_SPIN_PREPARE(obj);
1301 #endif /* USE_ITT_BUILD */
1302  __kmp_is_thread_alive(th, &exit_val);
1303  KMP_YIELD_OVERSUB_ELSE_SPIN(spins);
1304  } while (exit_val == STILL_ACTIVE && TCR_4(th->th.th_info.ds.ds_alive));
1305 #if USE_ITT_BUILD
1306  if (exit_val == STILL_ACTIVE) {
1307  KMP_FSYNC_CANCEL(obj);
1308  } else {
1309  KMP_FSYNC_SPIN_ACQUIRED(obj);
1310  }
1311 #endif /* USE_ITT_BUILD */
1312  }
1313 
1314  __kmp_free_handle(th->th.th_info.ds.ds_thread);
1315 
1316  /* NOTE: The ExitProcess(code) system call causes all threads to Terminate
1317  with a exit_val = code. Because of this we can not rely on exit_val having
1318  any particular value. */
1319  if (exit_val == STILL_ACTIVE) {
1320  KA_TRACE(1, ("__kmp_reap_common: thread still active.\n"));
1321  } else if ((void *)exit_val != (void *)th) {
1322  KA_TRACE(1, ("__kmp_reap_common: ExitProcess / TerminateThread used?\n"));
1323  }
1324 
1325  KA_TRACE(10,
1326  ("__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC
1327  "\n",
1328  th->th.th_info.ds.ds_gtid, th->th.th_info.ds.ds_thread));
1329 
1330  th->th.th_info.ds.ds_thread = 0;
1331  th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1332  th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1333  th->th.th_info.ds.ds_thread_id = 0;
1334 
1335  KMP_MB(); /* Flush all pending memory write invalidates. */
1336 }
1337 
1338 #if KMP_USE_MONITOR
1339 void __kmp_reap_monitor(kmp_info_t *th) {
1340  int status;
1341 
1342  KA_TRACE(10, ("__kmp_reap_monitor: try to reap %p\n",
1343  (void *)th->th.th_info.ds.ds_thread));
1344 
1345  // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1346  // If both tid and gtid are 0, it means the monitor did not ever start.
1347  // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1348  KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid);
1349  if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) {
1350  KA_TRACE(10, ("__kmp_reap_monitor: monitor did not start, returning\n"));
1351  return;
1352  }
1353 
1354  KMP_MB(); /* Flush all pending memory write invalidates. */
1355 
1356  status = SetEvent(__kmp_monitor_ev);
1357  if (status == FALSE) {
1358  DWORD error = GetLastError();
1359  __kmp_fatal(KMP_MSG(CantSetEvent), KMP_ERR(error), __kmp_msg_null);
1360  }
1361  KA_TRACE(10, ("__kmp_reap_monitor: reaping thread (%d)\n",
1362  th->th.th_info.ds.ds_gtid));
1363  __kmp_reap_common(th);
1364 
1365  __kmp_free_handle(__kmp_monitor_ev);
1366 
1367  KMP_MB(); /* Flush all pending memory write invalidates. */
1368 }
1369 #endif
1370 
1371 void __kmp_reap_worker(kmp_info_t *th) {
1372  KA_TRACE(10, ("__kmp_reap_worker: reaping thread (%d)\n",
1373  th->th.th_info.ds.ds_gtid));
1374  __kmp_reap_common(th);
1375 }
1376 
1377 #if KMP_HANDLE_SIGNALS
1378 
1379 static void __kmp_team_handler(int signo) {
1380  if (__kmp_global.g.g_abort == 0) {
1381  // Stage 1 signal handler, let's shut down all of the threads.
1382  if (__kmp_debug_buf) {
1383  __kmp_dump_debug_buffer();
1384  }
1385  KMP_MB(); // Flush all pending memory write invalidates.
1386  TCW_4(__kmp_global.g.g_abort, signo);
1387  KMP_MB(); // Flush all pending memory write invalidates.
1388  TCW_4(__kmp_global.g.g_done, TRUE);
1389  KMP_MB(); // Flush all pending memory write invalidates.
1390  }
1391 } // __kmp_team_handler
1392 
1393 static sig_func_t __kmp_signal(int signum, sig_func_t handler) {
1394  sig_func_t old = signal(signum, handler);
1395  if (old == SIG_ERR) {
1396  int error = errno;
1397  __kmp_fatal(KMP_MSG(FunctionError, "signal"), KMP_ERR(error),
1398  __kmp_msg_null);
1399  }
1400  return old;
1401 }
1402 
1403 static void __kmp_install_one_handler(int sig, sig_func_t handler,
1404  int parallel_init) {
1405  sig_func_t old;
1406  KMP_MB(); /* Flush all pending memory write invalidates. */
1407  KB_TRACE(60, ("__kmp_install_one_handler: called: sig=%d\n", sig));
1408  if (parallel_init) {
1409  old = __kmp_signal(sig, handler);
1410  // SIG_DFL on Windows* OS in NULL or 0.
1411  if (old == __kmp_sighldrs[sig]) {
1412  __kmp_siginstalled[sig] = 1;
1413  } else { // Restore/keep user's handler if one previously installed.
1414  old = __kmp_signal(sig, old);
1415  }
1416  } else {
1417  // Save initial/system signal handlers to see if user handlers installed.
1418  // 2009-09-23: It is a dead code. On Windows* OS __kmp_install_signals
1419  // called once with parallel_init == TRUE.
1420  old = __kmp_signal(sig, SIG_DFL);
1421  __kmp_sighldrs[sig] = old;
1422  __kmp_signal(sig, old);
1423  }
1424  KMP_MB(); /* Flush all pending memory write invalidates. */
1425 } // __kmp_install_one_handler
1426 
1427 static void __kmp_remove_one_handler(int sig) {
1428  if (__kmp_siginstalled[sig]) {
1429  sig_func_t old;
1430  KMP_MB(); // Flush all pending memory write invalidates.
1431  KB_TRACE(60, ("__kmp_remove_one_handler: called: sig=%d\n", sig));
1432  old = __kmp_signal(sig, __kmp_sighldrs[sig]);
1433  if (old != __kmp_team_handler) {
1434  KB_TRACE(10, ("__kmp_remove_one_handler: oops, not our handler, "
1435  "restoring: sig=%d\n",
1436  sig));
1437  old = __kmp_signal(sig, old);
1438  }
1439  __kmp_sighldrs[sig] = NULL;
1440  __kmp_siginstalled[sig] = 0;
1441  KMP_MB(); // Flush all pending memory write invalidates.
1442  }
1443 } // __kmp_remove_one_handler
1444 
1445 void __kmp_install_signals(int parallel_init) {
1446  KB_TRACE(10, ("__kmp_install_signals: called\n"));
1447  if (!__kmp_handle_signals) {
1448  KB_TRACE(10, ("__kmp_install_signals: KMP_HANDLE_SIGNALS is false - "
1449  "handlers not installed\n"));
1450  return;
1451  }
1452  __kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init);
1453  __kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init);
1454  __kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init);
1455  __kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init);
1456  __kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init);
1457  __kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init);
1458 } // __kmp_install_signals
1459 
1460 void __kmp_remove_signals(void) {
1461  int sig;
1462  KB_TRACE(10, ("__kmp_remove_signals: called\n"));
1463  for (sig = 1; sig < NSIG; ++sig) {
1464  __kmp_remove_one_handler(sig);
1465  }
1466 } // __kmp_remove_signals
1467 
1468 #endif // KMP_HANDLE_SIGNALS
1469 
1470 /* Put the thread to sleep for a time period */
1471 void __kmp_thread_sleep(int millis) {
1472  DWORD status;
1473 
1474  status = SleepEx((DWORD)millis, FALSE);
1475  if (status) {
1476  DWORD error = GetLastError();
1477  __kmp_fatal(KMP_MSG(FunctionError, "SleepEx()"), KMP_ERR(error),
1478  __kmp_msg_null);
1479  }
1480 }
1481 
1482 // Determine whether the given address is mapped into the current address space.
1483 int __kmp_is_address_mapped(void *addr) {
1484  DWORD status;
1485  MEMORY_BASIC_INFORMATION lpBuffer;
1486  SIZE_T dwLength;
1487 
1488  dwLength = sizeof(MEMORY_BASIC_INFORMATION);
1489 
1490  status = VirtualQuery(addr, &lpBuffer, dwLength);
1491 
1492  return !(((lpBuffer.State == MEM_RESERVE) || (lpBuffer.State == MEM_FREE)) ||
1493  ((lpBuffer.Protect == PAGE_NOACCESS) ||
1494  (lpBuffer.Protect == PAGE_EXECUTE)));
1495 }
1496 
1497 kmp_uint64 __kmp_hardware_timestamp(void) {
1498  kmp_uint64 r = 0;
1499 
1500  QueryPerformanceCounter((LARGE_INTEGER *)&r);
1501  return r;
1502 }
1503 
1504 /* Free handle and check the error code */
1505 void __kmp_free_handle(kmp_thread_t tHandle) {
1506  /* called with parameter type HANDLE also, thus suppose kmp_thread_t defined
1507  * as HANDLE */
1508  BOOL rc;
1509  rc = CloseHandle(tHandle);
1510  if (!rc) {
1511  DWORD error = GetLastError();
1512  __kmp_fatal(KMP_MSG(CantCloseHandle), KMP_ERR(error), __kmp_msg_null);
1513  }
1514 }
1515 
1516 int __kmp_get_load_balance(int max) {
1517  static ULONG glb_buff_size = 100 * 1024;
1518 
1519  // Saved count of the running threads for the thread balance algorithm
1520  static int glb_running_threads = 0;
1521  static double glb_call_time = 0; /* Thread balance algorithm call time */
1522 
1523  int running_threads = 0; // Number of running threads in the system.
1524  NTSTATUS status = 0;
1525  ULONG buff_size = 0;
1526  ULONG info_size = 0;
1527  void *buffer = NULL;
1528  PSYSTEM_PROCESS_INFORMATION spi = NULL;
1529  int first_time = 1;
1530 
1531  double call_time = 0.0; // start, finish;
1532 
1533  __kmp_elapsed(&call_time);
1534 
1535  if (glb_call_time &&
1536  (call_time - glb_call_time < __kmp_load_balance_interval)) {
1537  running_threads = glb_running_threads;
1538  goto finish;
1539  }
1540  glb_call_time = call_time;
1541 
1542  // Do not spend time on running algorithm if we have a permanent error.
1543  if (NtQuerySystemInformation == NULL) {
1544  running_threads = -1;
1545  goto finish;
1546  }
1547 
1548  if (max <= 0) {
1549  max = INT_MAX;
1550  }
1551 
1552  do {
1553 
1554  if (first_time) {
1555  buff_size = glb_buff_size;
1556  } else {
1557  buff_size = 2 * buff_size;
1558  }
1559 
1560  buffer = KMP_INTERNAL_REALLOC(buffer, buff_size);
1561  if (buffer == NULL) {
1562  running_threads = -1;
1563  goto finish;
1564  }
1565  status = NtQuerySystemInformation(SystemProcessInformation, buffer,
1566  buff_size, &info_size);
1567  first_time = 0;
1568 
1569  } while (status == STATUS_INFO_LENGTH_MISMATCH);
1570  glb_buff_size = buff_size;
1571 
1572 #define CHECK(cond) \
1573  { \
1574  KMP_DEBUG_ASSERT(cond); \
1575  if (!(cond)) { \
1576  running_threads = -1; \
1577  goto finish; \
1578  } \
1579  }
1580 
1581  CHECK(buff_size >= info_size);
1582  spi = PSYSTEM_PROCESS_INFORMATION(buffer);
1583  for (;;) {
1584  ptrdiff_t offset = uintptr_t(spi) - uintptr_t(buffer);
1585  CHECK(0 <= offset &&
1586  offset + sizeof(SYSTEM_PROCESS_INFORMATION) < info_size);
1587  HANDLE pid = spi->ProcessId;
1588  ULONG num = spi->NumberOfThreads;
1589  CHECK(num >= 1);
1590  size_t spi_size =
1591  sizeof(SYSTEM_PROCESS_INFORMATION) + sizeof(SYSTEM_THREAD) * (num - 1);
1592  CHECK(offset + spi_size <
1593  info_size); // Make sure process info record fits the buffer.
1594  if (spi->NextEntryOffset != 0) {
1595  CHECK(spi_size <=
1596  spi->NextEntryOffset); // And do not overlap with the next record.
1597  }
1598  // pid == 0 corresponds to the System Idle Process. It always has running
1599  // threads on all cores. So, we don't consider the running threads of this
1600  // process.
1601  if (pid != 0) {
1602  for (int i = 0; i < num; ++i) {
1603  THREAD_STATE state = spi->Threads[i].State;
1604  // Count threads that have Ready or Running state.
1605  // !!! TODO: Why comment does not match the code???
1606  if (state == StateRunning) {
1607  ++running_threads;
1608  // Stop counting running threads if the number is already greater than
1609  // the number of available cores
1610  if (running_threads >= max) {
1611  goto finish;
1612  }
1613  }
1614  }
1615  }
1616  if (spi->NextEntryOffset == 0) {
1617  break;
1618  }
1619  spi = PSYSTEM_PROCESS_INFORMATION(uintptr_t(spi) + spi->NextEntryOffset);
1620  }
1621 
1622 #undef CHECK
1623 
1624 finish: // Clean up and exit.
1625 
1626  if (buffer != NULL) {
1627  KMP_INTERNAL_FREE(buffer);
1628  }
1629 
1630  glb_running_threads = running_threads;
1631 
1632  return running_threads;
1633 } //__kmp_get_load_balance()
1634 
1635 // Functions for hidden helper task
1636 void __kmp_hidden_helper_worker_thread_wait() {
1637  KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
1638 }
1639 
1640 void __kmp_do_initialize_hidden_helper_threads() {
1641  KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
1642 }
1643 
1644 void __kmp_hidden_helper_threads_initz_wait() {
1645  KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
1646 }
1647 
1648 void __kmp_hidden_helper_initz_release() {
1649  KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
1650 }
1651 
1652 void __kmp_hidden_helper_main_thread_wait() {
1653  KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
1654 }
1655 
1656 void __kmp_hidden_helper_main_thread_release() {
1657  KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
1658 }
1659 
1660 void __kmp_hidden_helper_worker_thread_signal() {
1661  KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
1662 }
1663 
1664 void __kmp_hidden_helper_threads_deinitz_wait() {
1665  KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
1666 }
1667 
1668 void __kmp_hidden_helper_threads_deinitz_release() {
1669  KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
1670 }