
    wi1                     t    d dl Z d dlZd dlZd dlmZ d dlZd dlmZm	Z	m
Z
mZ d dlmZmZ  G d d          ZdS )    N)partial)DictOptionalListTuple)NonPickledSyncManagerformat_secondsc                   
   e Zd ZdZdej        j        dededdfdZ	deddfd	Z
d
edeeeeef                           fdZd
ededdfdZd
eddfdZ	 dd
ededeeeeef                           dedef
dZdefdZdefdZdS )WorkerInsightsz
    Worker insights class for profiling the worker start up time, waiting time and working time. When worker init and
    exit functions are provided it will time those as well.
    ctxn_jobsuse_dillreturnNc                     || _         || _        || _        d| _        d| _        d| _        d| _        d| _        d| _        d| _	        d| _
        d| _        d| _        d| _        dS )z
        Parameter class for worker insights.

        :param ctx: Multiprocessing context
        :param n_jobs: Number of workers
        :param use_dill: Whether dill is used as serialization library
        FN)r   r   r   insights_enabledinsights_managerinsights_manager_lockworker_start_up_timeworker_init_timeworker_n_completed_tasksworker_waiting_timeworker_working_timeworker_exit_timemax_task_durationmax_task_args)selfr   r   r   s       f/var/www/development/aibuddy-work/election-extract/venv/lib/python3.11/site-packages/mpire/insights.py__init__zWorkerInsights.__init__   s       !& !%%)" %)! !% )-% $(  $(  !% "& "    enable_insightsc                 R   |rt          | j                  | _        | j                                         | j                                        | _        | j                            t          j	        | j
        d          | _        | j                            t          j	        | j
        d          | _        | j                            t          j        | j
        d          | _        | j                            t          j	        | j
        d          | _        | j                            t          j	        | j
        d          | _        | j                            t          j	        | j
        d          | _        | j                            t          j	        | j
        dz  d          | _        | j                            dg| j
        z  dz            | _        nFd| _        d| _        d| _        d| _        d| _        d| _        d| _        d| _        d| _        d| _        || _        dS )zs
        Resets the insights containers

        :param enable_insights: Whether to enable worker insights
        F)lock    N)r   r   r   startr   Lockr   Arrayctypesc_doubler   r   r   c_intr   r   r   r   r   listr   r   )r   r    s     r   reset_insightszWorkerInsights.reset_insights=   s     	& %:$-$H$HD!!''))))-D&(,vZ_(`(`D%$(HNN6?DKV[N$\$\D!,0HNN6<[`N,a,aD)'+x~~fot{Y^~'_'_D$'+x~~fot{Y^~'_'_D$$(HNN6?DKV[N$\$\D!%)X^^FOT[ST_[`^%a%aD"!%!6!;!;RD4;<NQR<R!S!SD$(D!)-D&(,D%$(D!,0D)'+D$'+D$$(D!%)D"!%D /r   	worker_idc           
          | j         rs| j        5  | j        It          t	          | j        |dz  |dz   dz           | j        |dz  |dz   dz                               ndcddd           S # 1 swxY w Y   dS dS )z`
        Initialize insights for a specific worker

        :param worker_id: worker ID
        Nr#      )r   r   r   r+   zipr   r   r-   s     r   get_max_task_duration_listz)WorkerInsights.get_max_task_duration_list`   s      	I + I I  1= S!7	AyST}XYFY8Y!Z!%!3IM9q=TUBU4U!VX X Y Y YCGI I I I I I I I I I I I I I I I I I	I 	Is   AA..A25A2
start_timec                 T    | j         r t          j                    |z
  | j        |<   dS dS )zp
        Update start up time

        :param worker_id: Worker ID
        :param start_time: Timestamp
        N)r   timer   )r   r-   r3   s      r   update_start_up_timez#WorkerInsights.update_start_up_timen   s:       	L379;;3KD%i000	L 	Lr   c                 B    | j         r| j        |xx         dz  cc<   dS dS )zn
        Increment the number of completed tasks for this worker

        :param worker_id: Worker ID
        r/   N)r   r   r1   s     r   update_n_completed_tasksz'WorkerInsights.update_n_completed_tasksx   s>       	:))444944444	: 	:r   Fmax_task_duration_last_updatedmax_task_duration_listforce_updatec                     t          j                     }| j        rb|s	||z
  dk    rWt          | \  }}|| j        |dz  |dz   dz  <   | j        5  || j        |dz  |dz   dz  <   ddd           n# 1 swxY w Y   |}|S )a  
        Update synced containers with new top 5 max task duration + args. Updates every 2 seconds.

        :param worker_id: Worker ID
        :param max_task_duration_last_updated: Last updated timestamp
        :param max_task_duration_list: Local worker insights container that holds (task duration, task args) tuples,
            sorted for heapq
        :param force_update: Whether to force the update
        :return: Last updated timestamp
           r#   r/   N)r5   r   r0   r   r   r   )r   r-   r9   r:   r;   nowtask_durations	task_argss           r   update_task_insightsz#WorkerInsights.update_task_insights   s     ikk  	1l 	1s=[7[_`6`6`(+-C(D%NIJXD"9q=IMQ3F#FG+ T TJS"9q=IMQ3F#FGT T T T T T T T T T T T T T T-0*--s   A00A47A4c                 D   d }d }| j         si S t          t          d          } || j                  dd         ddd         }g g }}|D ]p}| j        |         dk    r n\| j        |         d	k    r'|                     || j        |                              |                    | j        |                    qt          | j                  }t          | j                  }	t          | j	                  }
t          | j
                  }t          | j                  }||	z   |
z   |z   |z   }t          t          | j                  t          t          || j                            t          t          || j                            t          t          || j	                            t          t          || j
                            t          t          || j                             ||           ||	           ||
           ||           ||          ||
          } ||          |d<   d|fd|	fd|
fd|fd|ffD ]U\  }} |t!          | d| d                    \  }}||dz   z  || d<    ||          || d<    ||          || d<   V|S )zt
        Creates insights from the raw insight data

        :return: dictionary containing worker insights
        c                 b    t          t          t          |                     | j                  S )z
            argsort, as to not be dependent on numpy, by
            https://stackoverflow.com/questions/3382352/equivalent-of-numpy-argsort-in-basic-python/3382369#3382369
            )key)sortedrangelen__getitem__)seqs    r   argsortz,WorkerInsights.get_insights.<locals>.argsort   s%    
 %C//s????r   c                     t          |           t          |           z  t          fd| D                       t          |           z  }t          j        |          }|fS )za
            Calculates mean and standard deviation, as to not be dependent on numpy
            c              3   >   K   | ]}t          |z
  d           V  dS )r=   N)pow).0x_means     r   	<genexpr>z@WorkerInsights.get_insights.<locals>.mean_std.<locals>.<genexpr>   s1      66Qs1u9a((666666r   )sumrG   mathsqrt)rI   _var_stdrP   s      @r   mean_stdz-WorkerInsights.get_insights.<locals>.mean_std   s_     HHs3xx'E6666#66666SAD9T??D$;r   T)with_millisecondsNr   r$   )n_completed_tasksstart_up_time	init_timewaiting_timeworking_time	exit_timetotal_start_up_timetotal_init_timetotal_waiting_timetotal_working_timetotal_exit_timetop_5_max_task_durationstop_5_max_task_args
total_timestart_upinitwaitingworkingexitworker__timeg:0yE>_ratio
_time_mean	_time_std)r   r   r	   r   r   appendrR   r   r   r   r   r   dictr+   r   mapgetattr)r   rJ   rW   format_seconds_func
sorted_idxrf   rg   idxra   rb   rc   rd   re   rh   insightsparttotalmeanstds                      r   get_insightszWorkerInsights.get_insights   sG   	@ 	@ 	@	 	 	 $ 	I%nMMM WT344RSS9$$B$?
8:B"5  	@ 	@C%c*a//!#&",,$++,?,?@VWZ@[,\,\]]]&&t'9#'>???? "$";<<d344 !9:: !9::d344(?:=OORddgvv
$t/L*M*M&*3/BDD]+^+^&_&_"&s+>@U'V'V"W"W%)#.A4C[*\*\%]%]%)#.A4C[*\*\%]%]"&s+>@U'V'V"W"W,?,?@S,T,T(;(;O(L(L+>+>?Q+R+R+>+>?Q+R+R(;(;O(L(L1I,?A A A "5!4Z!@!@ ()<=#_5&(:;&(:;#_5	7 	D 	DKD%
 !/D/D/D/D!E!EFFID#(-d1B(CH___%,?,?,E,EH((()+>+>s+C+CH'''((r   c                    | j         sdS |                                 }dddt          |d                    g}dD ]d}|                    d|                    dd	           d
|d| d          d|| d          d|| d          d|| d         dz  dd           e|d         dk     r|                    ddg           |                    g d           t          | j                  D ]}d| d|d         |          g}dD ]@}|                    |                    dd	           d|| d         |          d           A|                    d                    |                     |                    g d            t          t          |d!         |d"                   d#$          D ]%\  }\  }}|                    | d%| d|            &d&                    |          S )'zs
        Formats the worker insights_str and returns a string

        :return: worker insights_str string
        zSNo profiling stats available. Try to run a function first with insights enabled ...zWorkerPool insights-------------------z!Total number of tasks completed: r[   )ri   rj   rk   rl   rm   zTotal _ z time: total_ro   z	s (mean: rq   z, std: rr   z	, ratio: rp   g      Y@z.2fz%)working_ratiog?r$   z+Efficiency warning: working ratio is < 80%!)r$   zStats per workerz----------------zWorker zTasks completed: z: sz - )r$   zTop 5 longest tasksr   rf   rg   r/   )r%   z. Time: 
)r   r   rR   rs   replaceextendrF   r   join	enumerater0   )	r   rz   insights_strr{   r-   
worker_strtask_idxdurationargss	            r   get_insights_stringz"WorkerInsights.get_insights_string   s    $ 	ihh$$&&--`CI\@]<^<^``b
 G 	T 	TD !Sc3)?)? !S !SQeZ^QeQeQeHf !S !S)1T2E2E2E)F!S !SOW[_XjXjXjOk!S !S*2d???*Cd*J R!S !S !S T T T T
 O$s**!N!P Q Q Q 	 1 1 1 	2 	2 	2 t{++ 	8 	8I/I//Xh7J.KI.VXXZJJ g g!!T\\#s%;%;"e"ex4?WXa?b"e"e"effff

: 6 67777 	 4 4 4 	5 	5 	5 +4CA[8\8@AV8W5Y 5Y`a+c +c +c 	J 	J&H&x8 H HX H H$ H HIIIIyy&&&r   )F)__name__
__module____qualname____doc__multiprocessingcontextBaseContextintboolr   r,   r   r   r   floatstrr2   r6   r8   rA   r   r   r    r   r   r   r      s        
)"O3? )" )"X\ )"ae )" )" )" )"V!0d !0t !0 !0 !0 !0FIC IHT%PUWZPZJ[E\<] I I I ILc Lu L L L L L:# :$ : : : : 38. .c .SX .5=d5PSCT>U5V.+/.<A. . . ..Id I I I IV,'S ,' ,' ,' ,' ,' ,'r   r   )r(   rS   multiprocessing.contextr   	functoolsr   r5   typingr   r   r   r   mpire.utilsr   r	   r   r   r   r   <module>r      s                 . . . . . . . . . . . . = = = = = = = =D' D' D' D' D' D' D' D' D' D'r   