wiru,UddlZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z ddlm Z ddl mZmZmZmZmZmZmZddlmZddlmZdd lmZmZmZdd lmZmZm Z m!Z!dd lm"Z"m#Z#dd l$m%Z%m&Z&m'Z'dd l(m)Z)ddl*m+Z+ddl,m-Z-erddl.m/Z/ej0e1Z2dZ3dZ4gdZ5ee6e7d<dZ8dZ9dZ:dZ;dZddddddddd d!d"e?d#ee?e fd$e?d%ee?d&ee@d'eeee?e?fd(eeee?e?fd)ee6d*e@d+e6fd,ZAGd-d.ejBZCeeefZDGd/d0ZEd1eEd d!d"e?d$e?d%e?f d2ZFd1eEdeeeCeeDffd3ZGd4eDddfd5ZHd6eeDd d!d"e?d$e?d%e?ddf d7ZId6eeDd d!d"e?d$e?d%e?ddf d8ZJd6eeDd d!d"e?d$e?d%e?ddf d9ZKGd:d;eZLd4eDdeLfd<ZMd=d>deeDfd?ZNd=d>d@e6deeDfdAZOdBe?ddfdCZPdS)DN)datetime)Path)Lock) TYPE_CHECKINGAnyDictListOptionalTupleUnion)quote) constants)CommitOperationAdd UploadInfo_fetch_upload_modes)LocalUploadFileMetadataLocalUploadFilePathsget_local_upload_pathsread_upload_metadata)DEFAULT_REVISION REPO_TYPES)DEFAULT_IGNORE_PATTERNSfilter_repo_objectstqdm) _format_size)is_xet_available) sha_fileobj)HfApi d) 2Kr!}iiXiCOMMIT_SIZE_SCALEii'r#r" paths_listreturnctdt|tkr5tdt|ddtddddlm}|d}|D]}t|j}|j }t|j d krt|j nd }||d xxd z cc<t|d d D]K\}}|dkrd nd |d |}||d|L|D]o\} } | d } t| d} | | z} | t kr=| d krdn| }td|d| dd| dd| ddt dd pg}g}|D]t}|jj}|dz }|t(kr||j|fM|t,kr||j|fu|rd d|d dD}t|dkrdt|dz dnd}tdt|dt(d ||d!|rd d"|d dD}t|dkrdt|dz dnd}tdt|d#t,d$||d%td&d S)'au Validate upload against repository limits and warn about potential issues. Args: paths_list: List of file paths to be uploaded Warns about: - Too many files in the repository (>100k) - Too many entries (files or subdirectories) in a single folder (>10k) - Files exceeding size limits (>20GB recommended, >50GB hard limit) z/Running validation checks on files to upload...zYou are about to upload ,z. files. This exceeds the recommended limit of z files per repository. Consider: - Splitting your data into multiple repositories - Using fewer, larger files (e.g., parquet files) - See: https://huggingface.co/docs/hub/repositories-recommendationsr) defaultdictc$dtdS)Nr)filessubdirs)set|/var/www/development/aibuddy-work/election-extract/venv/lib/python3.11/site-packages/huggingface_hub/_upload_large_folder.pyz)_validate_upload_limits..XsqUXUZUZ=[=[r4.r0rN/r1rootzFolder 'z ' contains z entries (z files and z/ subdirectories). This exceeds the recommended z< entries per folder. Consider reorganising into sub-folders.iʚ;z - c3.K|]\}}|d|ddVdSz: z.1fGBNr3.0pathsizes r5 z*_validate_upload_limits..s:!a!ajdDT"9"9T"9"9"9"9!a!a!a!a!a!ar4z ... and z more filesFound z files exceeding the zGB hard limit: - zM These files may fail to upload. Consider splitting them into smaller chunks.c3.K|]\}}|d|ddVdSr<r3r>s r5rBz*_validate_upload_limits..s:!\!\jdDT"9"9T"9"9"9"9!\!\!\!\!\!\r4z files larger than zGB (recommended limit): - z2 Large files may slow down loading and processing.zValidation checks complete.)loggerinfolenMAX_FILES_PER_REPOwarning collectionsr.r path_in_repopartsstrparent enumeratejoinadditemsMAX_FILES_PER_FOLDER file_pathstatst_sizeMAX_FILE_SIZE_GBappendRECOMMENDED_FILE_SIZE_GB)r*r.entries_per_folderpathsr@rNrPichildfolderdata file_count subdir_count total_entriesfolder_display large_filesvery_large_filesrAsize_gb files_strmore_strs r5_validate_upload_limitsrk;s KKABBB :+++ Us: B U U5G c U U U   (''''')45[5[)\)\ = =E&'' &)%5%5%<%<T[!!!#6"7+++q0+++"%*-- = =HAuFFSSrr(;(;F v &y 1 5 5e < < < < = +0022   '] 4 ?++ "\1 / / /'-}}VV&N NN:>::mP::O::-9O::0D^:::   K>>##%%-& % % %  # #U%7$A B B B B / / /    2G< = = = MM!a!aL\]_^_]_L`!a!a!aaa LOP`LaLadeLeLeH#&6"7"7!";HHHHkm \S)** \ \AQ \ \ \& \ \ \    MM!\!\KXZYZXZO!\!\!\\\ GJ;GWGWZ[G[G[C#k"2"2Q"6CCCCac AS%% A A:R A A A& A A A    KK-.....r4T<)revisionprivateallow_patternsignore_patterns num_workers print_reportprint_report_everyapirrepo_id folder_path repo_typermrnrorprqrrrsctdtvrtdtttstdd|g}nt|tr|g}|tz }|(tj pd} t| dz d} |d } td | | jt#od j} | rt(nt*}t-fd dD||}fd|D}tdt1|dt3|tdfdt5|dD}t7||fdt9|D}|D]}|| r$t=dztAj } tAj!dtAj |z | kr6| r!tEtAj }#rtIjdn|D]}|%ttIjddS)zUpload a large folder to the Hub in the most resilient way possible. See [`HfApi.upload_large_folder`] for the full documentation. NzFor large uploads, `repo_type` is explicitly required. Please set it to `model`, `dataset` or `space`. If you are using the CLI, pass it as `--repo-type=model`.z"Invalid repo type, must be one of zProvided path: 'z' is not a directoryrT)rurwrnexist_okzRepo created: xetEnabled)rurwrmexpandc3K|]?}||V@dSN)is_file relative_toas_posix)r?r@rvs r5rBz/upload_large_folder_internal..sOiidZ^ZfZfZhZhi  + & & / / 1 1iiiiiir4z**/*)rorpc0g|]}t|Sr3)r)r?relpathrvs r5 z0upload_large_folder_internal..s$bbb7(g>>bbbr4rEz candidate files to uploadzStarting upload...c>g|]}|t|jfSr3)rrM)r?r]rvs r5rz0upload_large_folder_internal..s<     $[%2DEEF   r4zRecovering from metadata files)descc Rg|]#}tjtd$S))statusrtrurwrm)targetkwargs) threadingThread _worker_job)r?_rtrurwrmrs r5rz0upload_large_folder_internal..sY      "&$     r4z zIs done: exiting main loopzUpload is complete!)& ValueErrorrrr expanduserresolveis_dir isinstancerOros cpu_countmax create_reporGrHrur repo_info xet_enabledUPLOAD_BATCH_SIZE_XETUPLOAD_BATCH_SIZE_LFSrglobrIrkrLargeUploadStatusrangestartprintcurrent_reporttimesleep_print_overwriteis_doneloggingrR)rtrurvrwrmrnrorprqrrrsnb_coresrepo_urlis_xet_enabledupload_batch_sizefiltered_paths_listr*rTthreadsthreadlast_report_tsrs````` @r5upload_large_folder_internalrs& I    ""JjJJKKK#{##..0088::K     OMKMMMNNN OS ) ),*+..O<>>&Q(Q,** w)W_cddH KK+++,,,G   MM   2@Z--EZ.iiiik>N>Nv>V>Viii%' cbbbNabbbJ KKDZDDDEEEJ''' KK$%%%    *+KLLL   E u&7 8 8F        {##   G 0 fv,,...///Y[[N 1 9;; '+= = = : !6!6!8!8999!Y[[N >>    L5 6 6 6   KK%%''((( L&'''''r4ceZdZejZejZejZejZejZ dS) WorkerJobN) __name__ __module__ __qualname__enumautoSHA256GET_UPLOAD_MODE PREUPLOAD_LFSCOMMITWAITr3r4r5rrsO TY[[FdikkODIKKM TY[[F 49;;DDDr4rcleZdZdZddeedefdZdefdZde d ed e dd fd Z de fd Z de fdZd S)rzBContains information, queues and tasks for a large upload process.rrTrcJ||_tj|_tj|_tj|_tj|_t|_d|_ d|_ d|_ ||_ d|_ d|_d|_t!j|_d|_t|_|jD]}|\}}|j|j|)|j|j|K|jdkr"|js|j|x|js|j|t4d|jddS)NrrlfszSkipping file z! (already uploaded and committed))rTqueueQueue queue_sha256queue_get_upload_modequeue_preupload_lfs queue_commitrlocknb_workers_sha256nb_workers_get_upload_modenb_workers_preupload_lfsrnb_workers_commitnb_workers_waitinglast_commit_attemptrnow _started_at _chunk_idx _chunk_locksha256put upload_mode is_uploaded is_committedrGdebugrM)selfrTritemr]metadatas r5__init__zLargeUploadStatus.__init__%s 7<{}}@E ">Ckmm 7<{}}FF &'/0'-.%&7&''(48 #<>> !%J e eD"OE8&!%%d++++%-*..t4444%..x7K.(,,T2222* e!%%d++++ ce.@cccdddd e er4r+ch|j5t|jcdddS#1swxYwYdSr~)rr(rrs r5 target_chunkzLargeUploadStatus.target_chunkGsy   6 6$T_5 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6s '++successnb_itemsdurationNc |j5|s/td|d|xjdzc_nJ|t|jkr4|dkr.td|d|xjdz c_t dt|jttdz |_ddddS#1swxYwYdS)NzFailed to commit z9 files at once. Will retry with less files in next batch.r(zSuccessfully committed z. at once. Increasing the limit for next batch.r) rrGrKrr(rHrminrI)rrrrs r5 update_chunkzLargeUploadStatus.update_chunkKs0   W W %v8vvvwww1$.t???HrMM nhnnnooo1$!!S#>O:P:PST:T%U%UVVDO W W W W W W W W W W W W W W W W W WsB9CCCc d}d}d}d}d}d}d}d}d} d} d} |j5|jD]\} } | jr| dz } | | jz } | dz } | j|dz }|| jz }| jdkr|dz }| j|dz }| jr|dz }|| jz }| jr|dz }|| jz }t| }tj }| d}||j z }t|dd}d}|d|d |d z }|d z }|d z }|d |d| d t|d|d z }|d|d|d t|d|d z }|dkr |d|dz }|d|d| d t|d|d z }|d| dz }|dz }|d|jdz }|d|jdz }|d|jdz }|d|jdz }|d|jdz }|dz }|cdddS#1swxYwYdS)z4#9>>> >G O4+JOOO OG K)FKKK KG Ad&<AAA AG >4#:>>> >G x G_/ / / / / / / / / / / / / / / / / / sGG::G>G>c|j5td|jDcdddS#1swxYwYdS)Nc38K|]\}}|jp|jVdSr~rrr?rrs r5rBz,LargeUploadStatus.is_done..s1ee;1hx,F0Feeeeeer4)rallrTrs r5rzLargeUploadStatus.is_dones Y f feeZ^Zdeeeee f f f f f f f f f f f f f f f f f fs 377)r)rrr__doc__r JOB_ITEM_TintrrboolfloatrrOrrr3r4r5rr"sLL e ed:. e3 e e e eD6c6666 WD WC W5 WT W W W W=====~fffffffr4rrc d}t|}|dS|\}}|tjkr|d} t||j|nk#t $rt$rT} t d| tj |j |Yd} ~ nd} ~ wwxYw|j 5|xjdzc_dddn #1swxYwYn|tjkr t!|||||nQ#t $rt$r:} t d| tj Yd} ~ nd} ~ wwxYw|D]u}|\} } | jr| jdkr|j|5| jd kr|j|[|j|v|j 5|xjdzc_dddn #1swxYwYnn|tjkr t/||||||D]}|j|np#t $rt$rY} t d | tj |D]}|j|Yd} ~ nd} ~ wwxYw|j 5|xjdzc_dddn #1swxYwYn|tjkrt5j} d} t7|||||nr#t $rt$r[} t d | tj |D]}|j|d } Yd} ~ nd} ~ wwxYwt5j| z }|| t;|||j 5t5j|_|xjdzc_dddn #1swxYwYnX|tj krHt5j!tD|j 5|xj#dzc_#dddn #1swxYwY) a Main process for a worker. The worker will perform tasks based on the priority list until all files are uploaded and committed. If no tasks are available, the worker will wait for 10 seconds before checking again. If a task fails for any reason, the item(s) are put back in the queue for another worker to pick up. Read `upload_large_folder` docstring for more information on how tasks are prioritized. TNrzFailed to compute sha256: r)rtrurwrmzFailed to get upload mode: rregularzFailed to preupload LFS: zFailed to commit: F)$_determine_next_jobrr_compute_sha256rrKeyboardInterrupt ExceptionrGerror traceback format_excrrrr_get_upload_moderrrrrr_preupload_lfsrrr_commitrrIrrrrWAITING_TIME_IF_NO_TASKSr)rrtrurwrmnext_jobjobrTrerrstart_tsrrs r5rrs[/AE'v..   F U )" " "8D .%%%,006666$    . . . =!==>>>$&&&#''-------- .  . .((A-(( . . . . . . . . . . . . . . .I- - - ' CI`hiiiii$    ' ' ' >1>>???$&&&&&&&& ' ; ;" 8)'500.2248888)Y66'++D1111044T:::: 7 711Q611 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7I+ + + 9u#w)^fgggg!22D'++D11112$    9 9 9 <<<===$&&&!99D.224888899999 9  5 5//14// 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5I$ $ $y{{HG 39W_`````$       5!55666$&&&!22D'++D1111  y{{X-H   UX > > > . .-1Y[[*((A-(( . . . . . . . . . . . . . . .IN " " J/ 0 0 0 / /))Q.)) / / / / / / / / / / / / / / /u[/s)AC3A CCC..C25C2 D""E060E++E02HHH-3I!!K5AK  KK55K9<K9)L>>N-AN((N-1)P&&P*-P*"Q??RRcz|j5|jdkr|jdkr|jt j|jz dkri|xjdz c_t dtj t|j| fcdddS|jdkr|jdkri|xjdz c_t dtj t|j| fcdddS|j tkre|xjdz c_t dtdtjt|j tfcdddS|j|jkrg|jdkr\|xjdz c_t d tjt|j|jfcdddS|jdkra|jdkrV|xjdz c_t d tjt/|jfcdddS|j dkrg|jdkr\|xjdz c_t d tjt|j tfcdddS|j|jkrs|jdks t0js\|xjdz c_t d tjt|j|jfcdddS|jdkrV|xjdz c_t d tjt/|jfcdddS|j dkr\|xjdz c_t dtjt|j tfcdddS|jdkr\|xjdz c_t d tjt|j|jfcdddS|jdkr|jdkr|jt j|jz dkri|xjdz c_t dtj t|j| fcdddS|jdkr|jdkr|jdkr|j dkr|jdkr|jdkr|jdkrt|jdkri|xjdz c_t dtj t|j| fcdddSt5d|jDr(t d ddddS|xjdz c_t dt<dtjgfcdddS#1swxYwYdS)Nri,rz;Job: commit (more than 5 minutes since last commit attempt)zJob: commit (>100 files ready)zJob: get upload mode (>z files ready)z5Job: preupload LFS (no other worker preuploading LFS)z.Job: sha256 (no other worker computing sha256)z:Job: get upload mode (no other worker getting upload mode)zJob: preupload LFSz Job: sha256zJob: get upload moderlz-Job: commit (1 min since last commit attempt)z Job: commitc38K|]\}}|jp|jVdSr~rrs r5rBz&_determine_next_job..is1aa[Q&@(*@aaaaaar4z.All files have been processed! Exiting worker.zNo task available, waiting... (zs)) rrrqsizerrrGrrr_get_nrrMAX_NB_FILES_FETCH_UPLOAD_MODErrrrrrrrr_get_onerHF_HUB_ENABLE_HF_TRANSFERrrTrHrrr)rs r5rrsB i(i(  $ ) )#))++a//*6 f886AA  $ $ ) $ $ LLV W W W$fV-@&BUBUBWBW&X&XYi(i(i(i(i(i(i(i( % * *v/B/H/H/J/Jc/Q/Q  $ $ ) $ $ LL9 : : :$fV-@&BUBUBWBW&X&XY!i(i(i(i(i(i(i(i(& ) / / 1 15S S S  - - 2 - - LL`3Q``` a a a-vf6RTr/s/st-i(i(i(i(i(i(i(i(2 ' - - / /63K K KPVPostPtPt  + +q 0 + + LLP Q Q Q+VF4NPVPh-i-ij9i(i(i(i(i(i(i(i(> & & ( (1 , ,1IQ1N1N  $ $ ) $ $ LLI J J J$hv/B&C&CDEi(i(i(i(i(i(i(i(J ) / / 1 1A 5 5&:[_`:`:`  - - 2 - - LLU V V V-vf6RTr/s/stQi(i(i(i(i(i(i(i(X ' - - / /63K K K  +q 0 0 8[ 0  + +q 0 + + LL- . . .+VF4NPVPh-i-ijci(i(i(i(i(i(i(i(h & & ( (1 , ,  $ $ ) $ $ LL ' ' '$hv/B&C&CDoi(i(i(i(i(i(i(i(t ) / / 1 1A 5 5  - - 2 - - LL/ 0 0 0-vf6RTr/s/st{i(i(i(i(i(i(i(i(@ ' - - / /! 3 3  + +q 0 + + LL- . . .+VF4NPVPh-i-ijGi(i(i(i(i(i(i(i(N  $ ) )#))++a//*6 f886AA  $ $ ) $ $ LLH I I I$fV-@&BUBUBWBW&X&XY]i(i(i(i(i(i(i(i(f  $ ) )#))++a//#))++q00,224499*0022a77(A--1Q66/144  $ $ ) $ $ LL ' ' '$fV-@&BUBUBWBW&X&XY}i(i(i(i(i(i(i(i(BaaTZT`aaa a a ( KKH I I IGi(i(i(i(i(i(i(i(N  % % * % % LLW;SWWW X X XNB'Si(i(i(i(i(i(i(i(i(i(i(i(i(i(i(i(i(i(slB+^0B^0A:^0A<^0!A1^0A7^0#B^08A&^0+A,^0$A,^0B*^0C<^09^0#A^00^47^4rc|\}}|jX|jd5}t||_dddn #1swxYwY||dS)z1Compute sha256 of a file and save it in metadata.Nrb)rrVopenrhexsave)rr]rfs r5rrysOE8 _ ! !$ ' ' 31)!nn0022HO 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 MM%s'AA!ArTc <d|D}t||||t|d|jt ||D]C\}}|\}} |j| _|j| _|j | _ | |DdS)zmGet upload mode for each file and update metadata. Also receive info if the file should be ignored. c,g|]}t|Sr3_build_hacky_operationr?rs r5rz$_get_upload_mode..!@@@$'--@@@r4rD)safe) additionsrwruheadersrmendpointN) r_build_hf_headersr r-zip _upload_moder_should_ignorer _remote_oid remote_oidr") rTrtrurwrmr+radditionr]rs r5rrs A@%@@@I%%''xb))) eY//hx'4!)!8&2 e r4cd|D}||||||D]!\}}d|_||"dS)z(Preupload LFS files and update metadata.c,g|]}t|Sr3r&r(s r5rz"_preupload_lfs..r)r4)rurwrmr+TN)preupload_lfs_filesrr"rTrtrurwrmr+r]rs r5rrsz@@%@@@I !x# er4cd|D}|||||d|D]!\}}d|_||"dS)zCommit files to the repo.c,g|]}t|Sr3r&r(s r5rz_commit..r)r4z(Add files using upload-large-folder tool)rurwrm operationscommit_messageTN) create_commitrr"r8s r5rrs}@@%@@@IA !x $ er4ceZdZddZdS)HackyCommitOperationAddr+Ncpt|jtrt|j|_dSdSr~)rpath_or_fileobjrrOrs r5 __post_init__z%HackyCommitOperationAdd.__post_init__s: d*D 1 1 =#&t';#<#.s! > > >AIEIKK > > >r4)rrr)rrNs` r5rrs7 > > > >s;5;==!'<' > >>r4reportcJ|dz }tjjtfd|D}t |D]@}t jdt jdAt j|t jdt|dz zt j dS)zPrint a report, overwriting the previous lines. Since tqdm in using `sys.stderr` to (re-)write progress bars, we need to use `sys.stdout` to print the report. Note: works well only if no other process is writing to `sys.stdout`! rc3BK|]}t|zdzVdS)rN)rI)r?lineterminal_widths r5rBz#_print_overwrite..s3SSt3t99.2SSSSSSr4z zrr8N) shutilget_terminal_sizecolumnssum splitlinesrsysstdoutwriterIflush)rQnb_linesrrUs @r5rrs dNF-//7NSSSSv?P?P?R?RSSSSSH8__## $$$ """"JVJSNS1B1B1D1DR1H-I-IIJKKKJr4)QrrrrrVr[rrr rpathlibrrtypingrrrr r r r urllib.parser rDr _commit_apirrr _local_folderrrrrrrutilsrrrutils._cache_managerrutils._runtimer utils.sharhf_apir getLoggerrrGrrr(r__annotations__rrrJrUrYr[rkrOrrEnumrrrrrrrrrr?r'rrrr3r4r5rms   IIIIIIIIIIIIIIIIIILLLLLLLLLLvvvvvvvvvvvv33333333EEEEEEEEEE......,,,,,,""""""  8 $ $!$OOO49OOO]/-A(B]/t]/]/]/]/L#"6:7;!% t(t(t( t( t(sDy!t(  t( sm t(d^t(U49c>23t(eDIsN34t(#t(t(t(t(t(t(x ')@@ A ufufufufufufufufpj/ j/ j/j/ j/  j/j/j/j/Zj( 1j(huYPTU_P`E`?a6bj(j(j(j(d*D,7SUXdglp, $z*  3 SV be jn     4 # ' C C [^ cg    (=====0===  0G    &-$z2B?+??Z8H????STr4