
    gi8	                         d dl Z d dlmZ d dlmZ d dlmZ dZ e j        dd	                    e           de j
                  Zdd
ededee         fdZdS )    N)Iterator)commands)convert_symbol)inmmcmptemexpcbpddccspmuz
    (%[^\n]+) |                                 # comment
    (a-zA-Z) |                                  # letter
    ([_^])(\d) |                                # number succeeding an underscore or a caret
    (-?\d+(?:\.\d+)?\s*(?:|a>  )) | # dimension
    (\d+(?:\.\d+)?) |                           # integer/decimal
    (\.\d*) |                                   # dot (.) or decimal can start with just a dot
    (\\[\\\[\]{}\s!,:>;|_%#$&]) |             # escaped characters
    (\\(?:begin|end)\s*{[a-zA-Z]+\*?}) |      # begin or end
    (\\operatorname\s*{[a-zA-Z\s*]+\*?\s*}) | # operatorname
    #  color, fbox, href, hbox, mbox, style, text, textbf, textit, textrm, textsf, texttt
    (\\(?:color|fbox|hbox|href|mbox|style|text|textbf|textit|textrm|textsf|texttt))\s*{([^}]*)} |
    (\\[cdt]?frac)\s*([.\d])\s*([.\d])? |       # fractions
    (\\math[a-z]+)({)([a-zA-Z])(}) |          # commands starting with math
    (\\[a-zA-Z]+) |                             # other commands
    (\S)                                        # non-space character
    Tlatex_stringskip_commentsreturnc              #     K   t                               |           D ]=}t          t          d |                                                    }|d                             t          j                  r/d                    |          }t          |          }|r	d| dV  |D ]}|r|                    d          r n|
                    t                    r|                    dd          V  N|                    t          j        t          j        t          j        f          r+d                    |                    d                    V  |V  ?dS )	z
    Converts Latex string into tokens.

    :param latex_string: Latex string.
    :param skip_comments: Flag to skip comments (default=True).
    c                 
    | d uS )N )xs    n/var/www/development/aibuddy-work/election-extract/venv/lib/python3.11/site-packages/latex2mathml/tokenizer.py<lambda>ztokenize.<locals>.<lambda>'   s
         r    z&#x;% N)PATTERNfinditertuplefiltergroups
startswithr   MATHjoinr   endswithUNITSreplaceBEGINENDOPERATORNAMEsplit)r   r   matchtokens	full_mathsymbolcaptureds          r   tokenizer5      sf      !!,//  v55u||~~FFGG!9.. 	I#I..F %Fooo%%% 		 		H !4!4S!9!9   '' &&sB/////""HNHL(BW#XYY gghnnS1122222NNNN# r   )T)retypingr   latex2mathmlr   latex2mathml.symbols_parserr   r*   compiler(   VERBOSEr!   strboolr5   r   r   r   <module>r>      s    				       ! ! ! ! ! ! 6 6 6 6 6 6P
"* 88E??	  " J% , 3 t x}      r   