o
    8ήc                     @   s  d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlmZmZmZmZ ddlmZmZmZmZmZmZmZ ddlZddlmZ ddlm Z  ed Z!zddl"m#Z# e#  dd	l$m%Z% d
Z&W n e'y~   dZ&Y nw z
ddl$m(Z( d
Z)W n e'y   dZ)Y nw zddl*Z+d
Z,W n e'y   dZ,Y nw e
-e)dZ.e
-e&dZ/e)oe(j0dkZ1e
-e1dZ2ej34dZ5ej34dZ6ej7dkZ8e8Z9e5 Z:dd Z;e,rdd Z<ndd Z<dd Z=dd Z>G dd de?Z@G dd  d e@ZAG d!d" d"e?ZBG d#d$ d$e@ZCG d%d& d&e@ZDG d'd( d(e@ZEd)d* ZFd+d, ZGe	jHZIG d-d. d.e?ZJd/d0 ZKeGeIejLZMeGeKd1 ZNe5sQeGeKd2 ZOeGeKd3 ZPeGeKd4 ZQG d5d6 d6eZRe jSd7v ZTe
-eTd8G d9d: d:eRZUeUV  G d;d< d<e?ZWG d=d> d>eWeRZXeXV  G d?d@ d@eZYeG dAdB dBeYZZeZV  eG dCdD dDeYZ[eG dEdF dFeYZ\ee2G dGdH dHeYZ]ee/G dIdJ dJeYZ^eG dKdL dLeZ_ee.G dMdN dNeZ`eadOkre
b  dS dS )Pz
Tests the parallel backend
    N)jit	vectorizeguvectorizeset_num_threads)temp_directoryoverride_configTestCasetagskip_parfors_unsupported
linux_onlyneeds_external_compilers)_TIMEOUT)configg      N@)_check_tbb_version_compatible)tbbpoolTF)omppoolzOpenMP threadpool requiredzTBB threadpool requiredGNUzGNU OpenMP only testswindarwinl        c                 C      t | | S N)nponesnv r   H/tmp/pip-target-vg8gfxp4/lib/python/numba/tests/test_parallel_backend.pyfooH   s   r   c                 C   s2   t t | | ft | | f}|t |  | S r   )r   dotr   arange)r   r   xr   r   r   linalgM   s    r"   c                 C   r   r   )r   r    r   r   r   r   r"   Q   s   c                 C   s   | | S r   r   )abr   r   r   	ufunc_fooV   s   r%   c                 C   s   | | |d< d S )Nr   r   )r#   r$   outr   r   r   
gufunc_fooZ   s   r'   c                   @      e Zd Zdd ZdS )runnablec                 K   
   || _ d S r   )_options)selfoptionsr   r   r   __init___      
zrunnable.__init__N)__name__
__module____qualname__r.   r   r   r   r   r)   ^   s    r)   c                   @   r(   )
jit_runnerc                 C   B   t di | jt}d}d}t||}|||}tj|| d S N   
   r   )r   r+   r   r   testingassert_allcloser,   cfuncr#   r$   expectedgotr   r   r   __call__e      

zjit_runner.__call__Nr0   r1   r2   r>   r   r   r   r   r3   c       r3   c                   @      e Zd Zdd Zdd ZdS )mask_runnerc                 K   s   || _ || _d S r   )runnermask)r,   rD   rE   r-   r   r   r   r.   o   s   
zmask_runner.__init__c                 C   s   | j rt| j  |   d S r   )rE   r   rD   )r,   r   r   r   r>   s   s   
zmask_runner.__call__Nr0   r1   r2   r.   r>   r   r   r   r   rC   n   s    rC   c                   @   r(   )linalg_runnerc                 C   r4   r5   )r   r+   r"   r   r8   r9   r:   r   r   r   r>   }   r?   zlinalg_runner.__call__Nr@   r   r   r   r   rG   {   rA   rG   c                   @   r(   )vectorize_runnerc                 C   sV   t dgfi | jt}tjdtj }}t||}|||}tj|| d S )Nz(f4, f4)r7   )	r   r+   r%   r   randomastypefloat32r8   r9   r:   r   r   r   r>      s
   

zvectorize_runner.__call__Nr@   r   r   r   r   rH      rA   rH   c                   @   r(   )guvectorize_runnerc                 C   s\   dg}t |dfi | jt}tjdtj }}t||}|||}tj	|| d S )Nz(f4, f4, f4[:])z	(),()->()r7   )
r   r+   r'   r   rI   rJ   rK   r%   r8   r9   )r,   sigr;   r#   r$   r<   r=   r   r   r   r>      s   

zguvectorize_runner.__call__Nr@   r   r   r   r   rL      rA   rL   c              
   K   st   | d}zt  ttt| d D ]
}t| }|  qW d S  ty9 } z|	| W Y d }~d S d }~ww )Nqueueg      ?)
getfaulthandlerenablerangeintlenrI   choice	Exceptionput)fnlistkwargsq_fner   r   r   chooser   s   

r^   c                        fdd}|S )Nc                    s    }d|i fddt dD }|D ]}|  q|D ]}|  q| sLg }| s<||d | r0d}t|ddd |D  d S )	NrN   c                    s   g | ]
}t  fd qS ))targetargsrY   )r^   ).0i)rX   kwsparallel_classr   r   
<listcomp>   s    z8compile_factory.<locals>.run_compile.<locals>.<listcomp>r6   Fz)Error(s) occurred in delegated runner:
%s
c                 S   s   g | ]}t |qS r   )repr)rb   r!   r   r   r   rf      s    )rR   startjoinemptyappendrO   RuntimeError)rX   rZ   thstherrors_msgre   
queue_impl)rX   rd   r   run_compile   s"   

z$compile_factory.<locals>.run_compiler   )re   rs   rt   r   rr   r   compile_factory   s   ru   c                   @   rB   )_proc_class_implc                 C   r*   r   )_method)r,   methodr   r   r   r.      r/   z_proc_class_impl.__init__c                 O   s   t | j}|j|i |S r   )multiprocessingget_contextrw   Process)r,   ra   rY   ctxr   r   r   r>      s   z_proc_class_impl.__call__NrF   r   r   r   r   rv      s    rv   c                 C   s,   | dkrd } t | }t| }|j}||fS )Ndefault)ry   rz   rv   Queue)rx   r|   procrN   r   r   r   _get_mp_classes   s   
r   spawnfork
forkserverr}   c                   @   sh  e Zd ZdZeddedddedddeddedddeddedddeddddeddedddeddddgZe	s\eddd	edddd
eddd	edddd
gZ
ee
 ejdk rdg ZnddgZg ZeD ]ZeD ]
Zeeee qpqlddgZed ered ed ede	 d	gedddgedddgeedZddhZdddZdS )TestParallelBackendBasez6
    Base class for testing the parallel backends
    T)nopython)r   cache)r   nogilparallel)r   r`   )r   r`   r   )r   r   )r   r   r         	threadingrI   multiprocessing_spawnmultiprocessing_forkmultiprocessing_forkserver)concurrent_jitconcurrent_vectorizeconcurrent_guvectorizeconcurrent_mix_useconcurrent_mix_use_masksomptbbc                 C   s<  t | jj| _td| j |dkrt| nM|dkr t| nL|dkr)t| nK|dkr2t| nJ|dkr;t	| nI|dkr^ttg}t
rO|t |t t| |D ]}|| qVn.td| W d    d S W d    d S W d    d S W d    d S W d    d S W d    d S 1 sw   Y  d S )	N	CACHE_DIRr   r   r   r   multiprocessing_defaultrI   zUnknown parallelism supplied %s)r   	__class__r0   
_cache_dirr   thread_implfork_proc_implforkserver_proc_implspawn_proc_impldefault_proc_impl_HAVE_OS_FORKrl   rI   shuffle
ValueError)r,   rX   parallelismpsimplr   r   r   rt     sH   









"z#TestParallelBackendBase.run_compileN)r   )r0   r1   r2   __doc__r3   rG   rH   rL   	all_impls_parfors_unsupportedparfor_implsextendr   NUMBA_NUM_THREADSmasks
mask_implsr   rE   rl   rC   r   r   runnerssafe_backendsrt   r   r   r   r   r      sZ    













r   )r   r   	workqueuezThreading layer not explicitc                   @   s   e Zd ZdZedd ZdS )TestParallelBackendav   These are like the numba.tests.test_threadsafety tests but designed
    instead to torture the parallel backend.
    If a suitable backend is supplied via NUMBA_THREADING_LAYER these tests
    can be run directly. This test class cannot be run using the multiprocessing
    option to the test runner (i.e. `./runtests -m`) as daemon processes cannot
    have children.
    c                 C   sX   | j D ]&}| j D ]\}}d| d | }dd }|||}||_t| || q
qd S )Ntest_r[   c                    r_   )Nc                    s2   t  }|jrd}| | d S | j d d S )Nz)daemonized processes cannot have children)r   )ry   current_processdaemonskipTestrt   )r,   selfprocrq   r   pr   r   test_methodJ  s
   zBTestParallelBackend.generate.<locals>.methgen.<locals>.test_methodr   )r   r   r   r   r   r   methgenI  s   z-TestParallelBackend.generate.<locals>.methgen)r   r   itemsr0   setattr)clsr   namer   methnamer   r\   r   r   r   generateC  s   


zTestParallelBackend.generateN)r0   r1   r2   r   classmethodr   r   r   r   r   r   6  s    r   c                   @   s0   e Zd ZeeedddZdd Zdd Z	dS )	TestInSubprocessF r   r   r   c                 C   s~   t j|t jt j|d}tt|j}z'|  | \}}|j	dkr-t
d|j	| f | | fW |  S |  w )Nstdoutstderrenvr   /process failed with code %s: stderr follows
%s

subprocessPopenPIPEr   Timer_TEST_TIMEOUTkillri   communicate
returncodeAssertionErrordecodecancelr,   cmdliner   popentimeoutr&   errr   r   r   run_cmd`  s"   
zTestInSubprocess.run_cmdc                 C   s0   t j }t||d< tjdd|g}| ||S )NNUMBA_THREADING_LAYERz-mznumba.runtests)osenvironcopystrsys
executabler   )r,   testthreading_layerenv_copyr   r   r   r   run_test_in_separate_processs  s   
z-TestInSubprocess.run_test_in_separate_processN)
r0   r1   r2   skip_no_tbbskip_no_ompunittestskipIfbackendsr   r   r   r   r   r   r   [  s    
r   c                   @   s,   e Zd ZdZdZedd Zedd ZdS )TestSpecificBackenda  
    This is quite contrived, for each test in the TestParallelBackend tests it
    generates a test that will run the TestParallelBackend test in a new python
    process with an environment modified to ensure a specific threadsafe backend
    is used. This is with view of testing the backends independently and in an
    isolated manner such that if they hang/crash/have issues, it doesn't kill
    the test suite.
    Fc           
         sb   | j }tj}d| d | }d|||f  fdd}d|| f }	t| |	td|| d S )Nr   r[   z%s.%s.%sc                    sR   |   \}}| jrtd||f  | d| | d|v | d|v d S )Nzstdout:
 "%s"
 stderr:
 "%s"OKFAILERROR)r   _DEBUGprintassertIn
assertTrue)r,   or]   backendinjected_methodr   r   test_template  s   z2TestSpecificBackend._inject.<locals>.test_templateztest_%s_%s_%slong_running)r1   r   r0   r   r	   )
r   r   r   r   backend_guardthemodtheclsr   r   injected_testr   r   r   _inject  s   zTestSpecificBackend._injectc              	   C   st   | j  D ]2\}}| jD ]*}| j D ]"}|dv r$|dkr$tjdr$q|dv r-|dkr-q| |||| qqqd S )N)r   rI   r   linux)r   rI   r   )	r   r   r   r   keysr   platform
startswithr   )r   r   r   r   r   r   r   r   r     s"   

zTestSpecificBackend.generateN)r0   r1   r2   r   r   r   r   r   r   r   r   r   r   z  s    
r   c                   @   s6   e Zd ZdZdeje Zddei Z	dddZ
dS )	ThreadLayerTestHelperzP
    Helper class for running an isolated piece of code based on a template
    z%ra  if 1:
    import sys
    sys.path.insert(0, "%(here)r")
    import multiprocessing
    import numpy as np
    from numba import njit
    import numba
    try:
        import threading_backend_usecases
    except ImportError as e:
        print("DEBUG:", sys.path)
        raise e
    import os

    sigterm_handler = threading_backend_usecases.sigterm_handler
    busy_func = threading_backend_usecases.busy_func

    def the_test():
        %%s

    if __name__ == "__main__":
        the_test()
    hereNc                 C   s   |d u rt j }td|d< tj|tjtj|d}tt	|j
}z |  | \}}|jdkr<td|j| f W |  n|  w | | fS )Nr   r   r   r   r   )r   r   r   r   r   r   r   r   r   r   r   ri   r   r   r   r   r   r   r   r   r   r     s*   

zThreadLayerTestHelper.run_cmdr   )r0   r1   r2   r   r   pathdirname__file___heretemplater   r   r   r   r   r    s    r  c                   @   s@   e Zd ZdZdZeeedddZ	e
dd Ze
dd Zd	S )
TestThreadingLayerSelectionz@
    Checks that numba.threading_layer() reports correctly.
    Fr   r   c                    s0    fdd}d  }t | |td|| d S )Nc                    s^   d}| j |   }tjd|g}tj }t |d< | j||d\}}| jr-t	|| d S d S )Nzif 1:
                X = np.arange(1000000.)
                Y = np.arange(1000000.)
                Z = busy_func(X, Y)
                assert numba.threading_layer() == '%s'
            -cr   r   )
r  r   r   r   r   r   r   r   r   r   )r,   bodyrunmer   r   r&   r   r   r   r   r     s   
z:TestThreadingLayerSelection._inject.<locals>.test_templatez test_threading_layer_selector_%s	important)r   r	   )r   r   r   r   r   r   r  r   r     s
   z#TestThreadingLayerSelection._injectc                 C   s$   | j  D ]
\}}| || qd S r   )r   r   r   )r   r   r   r   r   r   r     s   z$TestThreadingLayerSelection.generateN)r0   r1   r2   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r	    s    

r	  c                   @   sZ   e Zd ZdefddZeedd Zeedd Zedd	 Z	ed
d Z
dd ZdS )TestThreadingLayerPriorityenv_varc                 C   sJ   t j }d|d< ||d< d| d}tjdt|g}| j||d dS )	zJTest setting priority via env var NUMBA_THREADING_LAYER_PRIORITY.
        r}   r   NUMBA_THREADING_LAYER_PRIORITYa  
                import numba

                # trigger threading layer decision
                # hence catching invalid THREADING_LAYER_PRIORITY
                @numba.jit(
                    'float64[::1](float64[::1], float64[::1])',
                    nopython=True,
                    parallel=True,
                )
                def plus(x, y):
                    return x + y

                captured_envvar = list("a	  ".split())
                assert numba.config.THREADING_LAYER_PRIORITY ==                     captured_envvar, "priority mismatch"
                assert numba.threading_layer() == captured_envvar[0],                    "selected backend mismatch"
                r
  r  N)r   r   r   r   r   textwrapdedentr   )r,   r  r   codecmdr   r   r   each_env_var  s   
z'TestThreadingLayerPriority.each_env_varc                 C   s0   g d}t |D ]}d|}| | q	d S )Nr    )	itertoolspermutationsrj   r  )r,   r}   r   r  r   r   r   test_valid_env_var3  s
   
z-TestThreadingLayerPriority.test_valid_env_varc                 C   sZ   d}|  t}| | W d    n1 sw   Y  dD ]}| | t|j qd S )Nztbb omp workqueue notvalidhere)z!THREADING_LAYER_PRIORITY invalid:zIt must be a permutation of)assertRaisesr   r  r   r   	exception)r,   r  raisesmsgr   r   r   test_invalid_env_var;  s   z/TestThreadingLayerPriority.test_invalid_env_varc                 C      dD ]}|  | qd S )N)zomp tbb workqueuezomp workqueue tbbr  r,   r  r   r   r   test_ompG     z#TestThreadingLayerPriority.test_ompc                 C   r!  )N)ztbb omp workqueueztbb workqueue ompr"  r#  r   r   r   test_tbbL  r%  z#TestThreadingLayerPriority.test_tbbc                 C   r!  )N)zworkqueue tbb ompzworkqueue omp tbbr"  r#  r   r   r   test_workqueueQ  s   z)TestThreadingLayerPriority.test_workqueueN)r0   r1   r2   r   r  r   r   r  r   r$  r&  r'  r   r   r   r   r    s    !


r  c                   @   sH   e Zd ZdZdZedd Zedd Zdd Z	e
ed	d
d ZdS )TestMiscBackendIssueszL
    Checks fixes for the issues with threading backends implementation
    Fc                 C   <   d}t jd|g}tj }d|d< d|d< | j||d dS )	z8
        Tests that OMP does not overflow stack
        a  if 1:
            from numba import vectorize, threading_layer
            import numpy as np

            @vectorize(['f4(f4,f4,f4,f4,f4,f4,f4,f4)'], target='parallel')
            def foo(a, b, c, d, e, f, g, h):
                return a+b+c+d+e+f+g+h

            x = np.ones(2**20, np.float32)
            foo(*([x]*8))
            assert threading_layer() == "omp", "omp not found"
        r
  r   r   100KOMP_STACKSIZEr  Nr   r   r   r   r   r   r,   r  r   r   r   r   r   test_omp_stack_overflow]  s   
z-TestMiscBackendIssues.test_omp_stack_overflowc                 C   r)  )	zq
        Tests that TBB works well with single thread
        https://github.com/numba/numba/issues/3440
        aM  if 1:
            from numba import njit, prange, threading_layer

            @njit(parallel=True)
            def foo(n):
                acc = 0
                for i in prange(n):
                    acc += i
                return acc

            foo(100)
            assert threading_layer() == "tbb", "tbb not found"
        r
  r   r   1r   r  Nr,  r-  r   r   r   test_single_thread_tbbt  s   
z,TestMiscBackendIssues.test_single_thread_tbbc              
   C   s   d}t jd|g}tj }d|d< d|d< z| j||d\}}W d
S  tyL } z| jr1t|| t	|}| 
d| | 
d	| W Y d
}~d
S d
}~ww )zW
        Tests workqueue raises sigabrt if a nested parallel call is performed
        a  if 1:
            from numba import njit, prange
            import numpy as np

            @njit(parallel=True)
            def nested(x):
                for i in prange(len(x)):
                    x[i] += 1


            @njit(parallel=True)
            def main():
                Z = np.zeros((5, 10))
                for i in prange(Z.shape[0]):
                    nested(Z[i])
                return Z

            main()
        r
  r   r   4r   r  zfailed with codez3Terminating: Nested parallel kernel launch detectedN)r   r   r   r   r   r   r   r   r   r   r   )r,   r  r   r   r&   r   r]   e_msgr   r   r   +test_workqueue_aborts_on_nested_parallelism  s"   

zATestMiscBackendIssues.test_workqueue_aborts_on_nested_parallelismzTest needs fork(2)c                 C   s<   d}t jd|g}tj }d|d< d|d< | j||d d S )Na  if 1:
            from numba import njit, prange, threading_layer
            import numpy as np
            import multiprocessing

            if __name__ == "__main__":
                # Need for force fork context (OSX default is "spawn")
                multiprocessing.set_start_method('fork')

                @njit(parallel=True)
                def func(x):
                    return 10. * x

                arr = np.arange(2.)

                # run in single process to start Numba's thread pool
                np.testing.assert_allclose(func(arr), func.py_func(arr))

                # now run in a multiprocessing pool to get a fork from a
                # non-main thread
                with multiprocessing.Pool(10) as p:
                    result = p.map(func, [arr])
                np.testing.assert_allclose(result,
                                           func.py_func(np.expand_dims(arr, 0)))

                assert threading_layer() == "workqueue"
        r
  r   r   r1  r   r  r,  r-  r   r   r   0test_workqueue_handles_fork_from_non_main_thread  s   
zFTestMiscBackendIssues.test_workqueue_handles_fork_from_non_main_threadN)r0   r1   r2   r   r   r   r.  r   r0  r3  r   
skipUnlessr   r4  r   r   r   r   r(  V  s    


(r(  c                   @   sT   e Zd ZdZdZdd Zdd Zdd Zed	d
 Z	dd Z
dd Zedd ZdS )TestForkSafetyIssueszV
    Checks Numba's behaviour in various situations involving GNU OpenMP and fork
    Fc                 C   s"   d}t jd|g}| |\}}d S )Nzsif 1:
            from numba.np.ufunc import omppool
            assert omppool.openmp_vendor == 'GNU'
            r
  )r   r   r   )r,   r  r   r&   r   r   r   r   !test_check_threading_layer_is_gnu  s   z6TestForkSafetyIssues.test_check_threading_layer_is_gnuc              
   C   sf   d}| j | }tjd|g}z
| |\}}W dS  ty2 } z| dt| W Y d}~dS d}~ww )z~
        Whilst normally valid, this actually isn't for Numba invariant of OpenMP
        Checks SIGABRT is received.
        zif 1:
            X = np.arange(1000000.)
            Y = np.arange(1000000.)
            Z = busy_func(X, Y)
            pid = os.fork()
            if pid  == 0:
                Z = busy_func(X, Y)
            else:
                os.wait()
        r
  zfailed with code -6N)r  r   r   r   r   r   r   )r,   r  r  r   r&   r   r]   r   r   r   !test_par_parent_os_fork_par_child  s   

z6TestForkSafetyIssues.test_par_parent_os_fork_par_childc                 C   @   d}| j | }tjd|g}| |\}}| jrt|| dS dS )au  
        Implicit use of multiprocessing fork context.
        Does this:
        1. Start with OpenMP
        2. Fork to processes using OpenMP (this is invalid)
        3. Joins fork
        4. Check the exception pushed onto the queue that is a result of
           catching SIGTERM coming from the C++ aborting on illegal fork
           pattern for GNU OpenMP
        a  if 1:
            mp = multiprocessing.get_context('fork')
            X = np.arange(1000000.)
            Y = np.arange(1000000.)
            q = mp.Queue()

            # Start OpenMP runtime on parent via parallel function
            Z = busy_func(X, Y, q)

            # fork() underneath with no exec, will abort
            proc = mp.Process(target = busy_func, args=(X, Y, q))
            proc.start()

            err = q.get()
            assert "Caught SIGTERM" in str(err)
        r
  Nr  r   r   r   r   r   r,   r  r  r   r&   r   r   r   r   *test_par_parent_implicit_mp_fork_par_child  s   
z?TestForkSafetyIssues.test_par_parent_implicit_mp_fork_par_childc                 C   r9  )au  
        Explicit use of multiprocessing fork context.
        Does this:
        1. Start with OpenMP
        2. Fork to processes using OpenMP (this is invalid)
        3. Joins fork
        4. Check the exception pushed onto the queue that is a result of
           catching SIGTERM coming from the C++ aborting on illegal fork
           pattern for GNU OpenMP
        a  if 1:
            X = np.arange(1000000.)
            Y = np.arange(1000000.)
            q = multiprocessing.Queue()

            # Start OpenMP runtime on parent via parallel function
            Z = busy_func(X, Y, q)

            # fork() underneath with no exec, will abort
            ctx = multiprocessing.get_context('fork')
            proc = ctx.Process(target = busy_func, args=(X, Y, q))
            proc.start()
            proc.join()

            err = q.get()
            assert "Caught SIGTERM" in str(err)
        r
  Nr:  r;  r   r   r   *test_par_parent_explicit_mp_fork_par_child'  s   
z?TestForkSafetyIssues.test_par_parent_explicit_mp_fork_par_childc                 C   r9  )z
        Explicit use of multiprocessing spawn, this is safe.
        Does this:
        1. Start with OpenMP
        2. Spawn to processes using OpenMP
        3. Join spawns
        4. Run some more OpenMP
        a  if 1:
            X = np.arange(1000000.)
            Y = np.arange(1000000.)
            q = multiprocessing.Queue()

            # Start OpenMP runtime and run on parent via parallel function
            Z = busy_func(X, Y, q)
            procs = []
            ctx = multiprocessing.get_context('spawn')
            for x in range(20): # start a lot to try and get overlap
                ## fork() + exec() to run some OpenMP on children
                proc = ctx.Process(target = busy_func, args=(X, Y, q))
                procs.append(proc)
                sys.stdout.flush()
                sys.stderr.flush()
                proc.start()

            [p.join() for p in procs]

            try:
                q.get(False)
            except multiprocessing.queues.Empty:
                pass
            else:
                raise RuntimeError("Queue was not empty")

            # Run some more OpenMP on parent
            Z = busy_func(X, Y, q)
        r
  Nr:  r;  r   r   r   -test_par_parent_mp_spawn_par_child_par_parentJ  s   	
zBTestForkSafetyIssues.test_par_parent_mp_spawn_par_child_par_parentc                 C   r9  )a  
        Implicit use of multiprocessing (will be fork, but cannot declare that
        in Py2.7 as there's no process launch context).
        Does this:
        1. Start with no OpenMP
        2. Fork to processes using OpenMP
        3. Join forks
        4. Run some OpenMP
        a,  if 1:
            X = np.arange(1000000.)
            Y = np.arange(1000000.)
            q = multiprocessing.Queue()

            # this is ok
            procs = []
            for x in range(10):
                # fork() underneath with but no OpenMP in parent, this is ok
                proc = multiprocessing.Process(target = busy_func,
                                               args=(X, Y, q))
                procs.append(proc)
                proc.start()

            [p.join() for p in procs]

            # and this is still ok as the OpenMP happened in forks
            Z = busy_func(X, Y, q)
            try:
                q.get(False)
            except multiprocessing.queues.Empty:
                pass
            else:
                raise RuntimeError("Queue was not empty")
        r
  Nr:  r;  r   r   r   =test_serial_parent_implicit_mp_fork_par_child_then_par_parentv     

zRTestForkSafetyIssues.test_serial_parent_implicit_mp_fork_par_child_then_par_parentc                 C   r9  )z
        Explicit use of multiprocessing 'fork'.
        Does this:
        1. Start with no OpenMP
        2. Fork to processes using OpenMP
        3. Join forks
        4. Run some OpenMP
        a'  if 1:
            X = np.arange(1000000.)
            Y = np.arange(1000000.)
            q = multiprocessing.Queue()

            # this is ok
            procs = []
            ctx = multiprocessing.get_context('fork')
            for x in range(10):
                # fork() underneath with but no OpenMP in parent, this is ok
                proc = ctx.Process(target = busy_func, args=(X, Y, q))
                procs.append(proc)
                proc.start()

            [p.join() for p in procs]

            # and this is still ok as the OpenMP happened in forks
            Z = busy_func(X, Y, q)
            try:
                q.get(False)
            except multiprocessing.queues.Empty:
                pass
            else:
                raise RuntimeError("Queue was not empty")
        r
  Nr:  r;  r   r   r   =test_serial_parent_explicit_mp_fork_par_child_then_par_parent  r@  zRTestForkSafetyIssues.test_serial_parent_explicit_mp_fork_par_child_then_par_parentN)r0   r1   r2   r   r   r7  r8  r<  r   r=  r>  r?  rA  r   r   r   r   r6    s    !
",)r6  c                   @   s,   e Zd ZdZedd Zeedd ZdS )TestTBBSpecificIssuesFc                 C   sP   d}t jd|g}| |\}}d}| || | jr&td| td| d S d S )Na%  if 1:
            import threading
            import numba
            numba.config.THREADING_LAYER='tbb'
            from numba import njit, prange, objmode
            from numba.core.serialize import PickleCallableByPath
            import os

            e_running = threading.Event()
            e_proceed = threading.Event()

            def indirect_core():
                e_running.set()
                # wait for forker() to have forked
                while not e_proceed.isSet():
                    pass

            indirect = PickleCallableByPath(indirect_core)

            @njit
            def obj_mode_func():
                with objmode():
                    indirect()

            @njit(parallel=True, nogil=True)
            def work():
                acc = 0
                for x in prange(10):
                    acc += x
                obj_mode_func()
                return acc

            def runner():
                work()

            def forker():
                # wait for the jit function to say it's running
                while not e_running.isSet():
                    pass
                # then fork
                os.fork()
                # now fork is done signal the runner to proceed to exit
                e_proceed.set()

            numba_runner = threading.Thread(target=runner,)
            fork_runner =  threading.Thread(target=forker,)

            threads = (numba_runner, fork_runner)
            for t in threads:
                t.start()
            for t in threads:
                t.join()
        r
  z9Attempted to fork from a non-main thread, the TBB libraryOUT:ERR:)r   r   r   r   r   r   )r,   r  r   r&   r   msg_headr   r   r   test_fork_from_non_main_thread  s   6
z4TestTBBSpecificIssues.test_fork_from_non_main_threadc                 C   s   d}d d|}tjd|g}tj }d|d< | j||d\}}||v r*| d n| d	| | j	r?t
d
| t
d| d S d S )NzSKIP: COMPILATION FAILEDar  if 1:
            import ctypes
            import sys
            import multiprocessing as mp
            from tempfile import TemporaryDirectory, NamedTemporaryFile
            from numba.pycc.platform import Toolchain, _external_compiler_ok
            from numba import njit, prange, threading_layer
            import faulthandler
            faulthandler.enable()
            if not _external_compiler_ok:
                raise AssertionError('External compilers are not found.')
            with TemporaryDirectory() as tmpdir:
                with NamedTemporaryFile(dir=tmpdir) as tmpfile:
                    try:
                        src = """
                        #define TBB_PREVIEW_WAITING_FOR_WORKERS 1
                        #include <tbb/tbb.h>
                        static tbb::task_scheduler_handle tsh;
                        extern "C"
                        {
                        void launch(void)
                        {
                            tsh = tbb::task_scheduler_handle::get();
                        }
                        }
                        """
                        cxxfile = f"{tmpfile.name}.cxx"
                        with open(cxxfile, 'wt') as f:
                            f.write(src)
                        tc = Toolchain()
                        object_files = tc.compile_objects([cxxfile,],
                                                           output_dir=tmpdir)
                        dso_name = f"{tmpfile.name}.so"
                        tc.link_shared(dso_name, object_files,
                                       libraries=['tbb',],
                                       export_symbols=['launch'])
                        # Load into the process, it doesn't matter whether the
                        # DSO exists on disk once it's loaded in.
                        DLL = ctypes.CDLL(dso_name)
                    except Exception as e:
                        # Something is broken in compilation, could be one of
                        # many things including, but not limited to: missing tbb
                        # headers, incorrect permissions, compilers that don't
                        # work for the above
                        print(e)
                        print('BROKEN_COMPILERS')
                        sys.exit(0)

                    # Do the test, launch this library and also execute a
                    # function with the TBB threading layer.

                    DLL.launch()

                    @njit(parallel=True)
                    def foo(n):
                        acc = 0
                        for i in prange(n):
                            acc += i
                        return acc

                    foo(1)

            # Check the threading layer used was TBB
            assert threading_layer() == 'tbb'

            # Use mp context for a controlled version of fork, this triggers the
            # reported bug.

            ctx = mp.get_context('fork')
            def nowork():
                pass
            p = ctx.Process(target=nowork)
            p.start()
            p.join(10)
            print("SUCCESS")
            BROKEN_COMPILERSr
  r   r   r  z3Compilation of DSO failed. Check output for detailsSUCCESSrC  rD  )replacer   r   r   r   r   r   r   r   r   r   )r,   rG  r  r   r   r&   r   r   r   r   &test_lifetime_of_task_scheduler_handle*  s   KM

z<TestTBBSpecificIssues.test_lifetime_of_task_scheduler_handleN)r0   r1   r2   r   r   rF  r   rJ  r   r   r   r   rB    s    
ZrB  c                   @   s,   e Zd ZdZdd Zedd Zdd ZdS )	TestInitSafetyIssuesFc                 C   s~   t j|t jt jd}tt|j}z |  | \}}|j	dkr,t
d|j	| f W |  n|  w | | fS )N)r   r   r   r   r   )r,   r   r   r   r&   r   r   r   r   r     s"   
zTestInitSafetyIssues.run_cmdc                 C   s\   t jt jtd}tj|g}| |\}}| d| | j	r,t
d| t
d| d S d S )Nzorphaned_semaphore_usecase.pyzleaked semaphorerC  rD  )r   r  rj   r  r  r   r   r   assertNotInr   r   )r,   	test_filer   r&   r   r   r   r   test_orphaned_semaphore  s   

z,TestInitSafetyIssues.test_orphaned_semaphorec              	   C   sz   dD ]8}zt | W n	 ty   Y qw d}tjd||g}| |\}}| jr4td| td| | 	|| qd S )N)r   r   r   zimport numba; import multiprocessing;multiprocessing.set_start_method('{}');print(multiprocessing.get_context().get_start_method())r
  rC  rD  )
ry   rz   r   r   r   formatr   r   r   r   )r,   methr  r   r&   r   r   r   r   test_lazy_lock_init  s   

z(TestInitSafetyIssues.test_lazy_lock_initN)r0   r1   r2   r   r   r   rN  rQ  r   r   r   r   rK    s    
rK  c                   @   r(   )TestOpenMPVendorsc                 C   sN   t  }d|d< d|d< d|d< | D ]}tj|r$| || tj qdS )z>
        Checks the OpenMP vendor strings are correct
        MSwin32Intelr   r   r   N)dictr   r   r   r  assertEqualr   openmp_vendor)r,   r<   kr   r   r   test_vendors  s   zTestOpenMPVendors.test_vendorsN)r0   r1   r2   rZ  r   r   r   r   rR    s    rR  __main__)cr   rP   r  ry   r   rI   r   r   r  r   r   numpyr   numbar   r   r   r   numba.tests.supportr   r   r   r	   r
   r   r   rN   t_queuenumba.testing.mainr   _RUNNER_TIMEOUT
numba.corer   r   numba.np.ufunc.parallelr   numba.np.ufuncr   _HAVE_TBB_POOLImportErrorr   _HAVE_OMP_POOLscipy.linalg.cython_lapackscipy_HAVE_LAPACKr5  r   r   rX  _gnuompskip_unless_gnu_ompr   r  _windows_osxmaxsize_32bitr   r   r   r"   r%   r'   objectr)   r3   rC   rG   rH   rL   r^   ru   Thread_thread_classrv   r   r~   r   r   r   r   r   r   THREADING_LAYER_specific_backendsr   r   r   r   r  r	  r  r(  r6  rB  rK  rR  r0   mainr   r   r   r   <module>   s   $



	
X
!36%F 
 h A<
