PidddlZddlZddlZddlZddlZddlZddlmZddlmZddl m Z ddl m Z ddl mZddlmZejejeZejedgZejed Zd gZdaejd Zejd ZGd deZdZddddddZ ddddddZ!dZ"e#e"Z$dZ%e&de'dDZ(de(d<de(d<de(d<dZ)d Z*Gd!d"eZ+Gd#d$eZ,dS)%N)Path)knobs)compile_module_from_src) _allocation) GPUTarget) GPUDriverincludelibcudactjjx}r|gStjddgd}d|D}d|D}tjd}|r!|sd| d D}d }|r|d t|zz }|d z }n |d z }|dz }td|Ds J||S)Nz/sbin/ldconfigz-pignore)errorscJg|] }d|v|d!S) libcuda.so.1)split).0lines u/var/www/development/aibuddy-work/election-extract/venv/lib/python3.11/site-packages/triton/backends/nvidia/driver.py z libcuda_dirs..s0 U U UnPT>T>TDJJLL >T>T>TcLg|]!}tj|"S)ospathdirname)rlocs rrz libcuda_dirs..s& 1 1 1SBGOOC 1 1 1rLD_LIBRARY_PATHcg|]A}tjtj|d?|BS)rrrexistsjoin)rdirs rrz libcuda_dirs..!sAsssPRPWP\P\]`bpPqPqArArssssr:zlibcuda.so cannot found! z!Possible files are located at %s.z:Please create a symlink of libcuda.so to any of the files.zzlibcuda_dirs..)s@SSdrw~~bgll4@@AASSSSSSr) rnvidia libcuda_path subprocess check_outputdecode splitlinesrgetenvrstrany)env_libcuda_pathlibslocsdirsenv_ld_library_pathmsgs r libcuda_dirsr6s3 <44" !!  "$4d#; < < C C8 C T TD V U):): U U UD 1 1D 1 1 1D)$566t4tss288==sss &C ? 2SYY>> KK MM >> SSdSSS S SXXUXXXX Krc.tgtSN) libdevice_dirr6rrr library_dirsr:-s  +LNN ++rc$eZdZfdZdZxZS) CudaUtilsct|ds-tt|||_|jS)Ninstance)hasattrsuperr<__new__r>)cls __class__s rrAzCudaUtils.__new__9s<sJ'' > C0088==CL|rcdtttjt ddttt}|j a |j |_ |j |_ |j |_ |j|_|j|_dS)Nzdriver.c cuda_utilssrcnamer: include_dirs libraries)rrrrr"r read_textr:rIrJ PyCUtensorMap load_binaryget_device_propertiescuOccupancyMaxActiveClustersset_printf_fifo_sizefill_tma_descriptor)selfmods r__init__zCudaUtils.__init__>s%RW\\':6677AACC%%    ) ?%(%>",/,L)$'$<!#&#:   r)__name__ __module__ __qualname__rArT __classcell__rCs@rr<r<7sG ;;;;;;;rr<c|ddkrdS|drdSiddddd d d d d ddddddddddddddddddddddd|S)Nr* CUdeviceptr tensordesc CUtensorMapi1int8_ti8i16int16_ti32int32_ti64int64_tu1uint8_tu8u16uint16_tu32uint32_tu64uint64_tfp16doublebf16fp32f32fp64 nvTmaDesc) startswith)tys r ty_to_cpprzTs !u||} }}\""}  h  h  y  y   y   i   i  z  z  z        x    ]! " #  rrlrnrp)rqrsrtrurv pack_fp16 pack_bf16 pack_fp32 pack_fp64 iiiKKppOOOOOOcfd}fdfdfd||}dt|D}dfd|D}t|z}g}|D]}||dt|D}t |d kr4d d d |Dznd} g} |D]b\} } | d kr | t vr&| t | d | ;| t| d | cd | } g}|D]\} } | d dkr|d| d+| t vr|d| dN| dkr|d| m| d kr|d| tt |}d}d|D}d|D}d|D}d|D}|d|ddt | d krd | znddd |d|fd|Dd |d!| d"||d||d||d#t |d krd d |zndd$}|S)%Ncg}d}|D]k}t|tr=|dr' r |nd}|dz }tjd|}|d}|d}|ddz}|X|d|ztd|zD]} |d|d n|d t|D]} |d t|D]} |dV||m r|t ksJ|S) Nrr]ztensordesc<([^[>]*)\[([^]]*)\],r[rfr_rwrd) isinstancer.rxrematchgroupcountappendrangelen) signatureoutputtensordesc_idxsigmetardtypeshapendim_tensordesc_metas r_expand_signaturez(make_launcher.._expand_signatures # #C#s## #|(D(D #:IS~66t!#!CSII A A{{3''!+<MM#+... #1t8__-- e,,,,MM$''''MM+...t))AMM%((((t))AMM%(((() c"""""LnO8L8L&L&L&L&L rct|tr|D]}||dS||dSr8)rtupler)rrx_flatten_signatures rrz)make_launcher.._flatten_signatures^ c5 ! !  . .""1f---- . . MM#     rct|tr)dt|}d|dS|ddkrdS|dvrdSt |S)Nr[]rr[z PyObject* constexprrw)rrr"maprz)ryval_extracted_types rrz&make_launcher.._extracted_typesm b%  ((33344Cs:::  a5C<<; + + +;}}rc t|tr)dt|}d|dS|ddkrdS|dvrdS|drdSd d d d d dddddd t |S)N()rr[Orr]dlbhiLBHIK) rrlongr`rcrergrirlrnrp)rrr"rrxrz)ryr format_ofs rrz make_launcher..format_ofs b%  ''#i,,--Cs:::  a5C<<3 + + +3 == & & 3    B--  rci|]\}}|| Srrrrss r z!make_launcher..s>>>$!QA>>>rrc&g|] }|Srr)rryrs rrz!make_launcher..s!FFFR99R==FFFrci|]\}}|| Srrrs rrz!make_launcher..s<<<$!QA<<.s, L LB L L L L L Lrrz argr[ptr_infoz.dev_ptr_arg_storagerwz*tma_ptrz c Jg|] \}}|ddkd|d|d|d|d !S)rr[zDevicePtrInfo ptr_infoz = getPointer(_argrz); if (!ptr_infoz.valid) return NULL;rrs rrz!make_launcher..sU Ar a5C<< feeaee1eeaeee <.sO\a\]_a    XqWWAWWaWWW   rc ng|]2\}}|tvt|d|dt|d|d3S) _argz _storage = z(_argz);)FLOAT_STORAGE_TYPEFLOAT_PACK_FUNCTIONrs rrz!make_launcher..s_ Ar # # # b !YYYY6I"6MYYTUYYY # # #rc*g|]\}}|dk d|S)rz&argrrs rrz!make_launcher..s, M M MUQ2;L;LjQjj;L;L;Lrz&global_scratchz&profile_scratcha #include "cuda.h" #include #include #include #define PY_SSIZE_T_CLEAN #include typedef struct { PyObject_HEAD; _Alignas(128) CUtensorMap tensorMap; } PyCUtensorMapObject; static inline void gpuAssert(CUresult code, const char *file, int line) { if (code != CUDA_SUCCESS) { const char* prefix = "Triton Error [CUDA]: "; const char* str; cuGetErrorString(code, &str); char err[1024] = {0}; strcat(err, prefix); strcat(err, str); PyGILState_STATE gil_state; gil_state = PyGILState_Ensure(); PyErr_SetString(PyExc_RuntimeError, err); PyGILState_Release(gil_state); } } #define CUDA_CHECK(ans) { gpuAssert((ans), __FILE__, __LINE__); } typedef CUresult (*cuLaunchKernelEx_t)(const CUlaunchConfig* config, CUfunction f, void** kernelParams, void** extra); static cuLaunchKernelEx_t getLaunchKernelExHandle() { // Open the shared library void* handle = dlopen("libcuda.so.1", RTLD_LAZY); if (!handle) { PyErr_SetString(PyExc_RuntimeError, "Failed to open libcuda.so.1"); return NULL; } // Clear any existing error dlerror(); cuLaunchKernelEx_t cuLaunchKernelExHandle = (cuLaunchKernelEx_t)dlsym(handle, "cuLaunchKernelEx"); // Check for errors const char *dlsym_error = dlerror(); if (dlsym_error) { PyErr_SetString(PyExc_RuntimeError, "Failed to retrieve cuLaunchKernelEx from libcuda.so.1"); return NULL; } return cuLaunchKernelExHandle; } static void _launch(int gridX, int gridY, int gridZ, int num_warps, int num_ctas, int launch_cooperative_grid, int launch_pdl, int clusterDimX, int clusterDimY, int clusterDimZ, int shared_memory, CUstream stream, CUfunction function, CUdeviceptr global_scratch, CUdeviceptr profile_scratchz) { void *params[] = { a }; if (gridX*gridY*gridZ > 0) { // 4 attributes that we can currently pass maximum CUlaunchAttribute launchAttr[4]; static cuLaunchKernelEx_t cuLaunchKernelExHandle = NULL; if (cuLaunchKernelExHandle == NULL) { cuLaunchKernelExHandle = getLaunchKernelExHandle(); } CUlaunchConfig config; config.gridDimX = gridX; config.gridDimY = gridY; config.gridDimZ = gridZ; if (num_ctas != 1) { config.gridDimX *= clusterDimX; config.gridDimY *= clusterDimY; config.gridDimZ *= clusterDimZ; } config.blockDimX = 32 * num_warps; config.blockDimY = 1; config.blockDimZ = 1; config.sharedMemBytes = shared_memory; config.hStream = stream; config.attrs = launchAttr; int num_attrs = 0; if (launch_pdl != 0) { CUlaunchAttribute pdlAttr = { .id = CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_STREAM_SERIALIZATION, .value = 1}; launchAttr[num_attrs] = pdlAttr; ++num_attrs; } if (launch_cooperative_grid != 0) { CUlaunchAttribute coopAttr = { .id = CU_LAUNCH_ATTRIBUTE_COOPERATIVE, .value = 1}; launchAttr[num_attrs] = coopAttr; ++num_attrs; } if (num_ctas != 1) { CUlaunchAttribute clusterAttr = {}; clusterAttr.id = CU_LAUNCH_ATTRIBUTE_CLUSTER_DIMENSION; clusterAttr.value.clusterDim.x = clusterDimX; clusterAttr.value.clusterDim.y = clusterDimY; clusterAttr.value.clusterDim.z = clusterDimZ; launchAttr[num_attrs] = clusterAttr; ++num_attrs; CUlaunchAttribute clusterSchedulingAttr = {}; clusterSchedulingAttr.id = CU_LAUNCH_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE; clusterSchedulingAttr.value.clusterSchedulingPolicyPreference = CU_CLUSTER_SCHEDULING_POLICY_SPREAD; launchAttr[num_attrs] = clusterSchedulingAttr; ++num_attrs; } config.numAttrs = num_attrs; CUDA_CHECK(cuLaunchKernelExHandle(&config, function, params, 0)); } } typedef struct _DevicePtrInfo { CUdeviceptr dev_ptr; bool valid; } DevicePtrInfo; static PyObject* data_ptr_str = NULL; static PyObject* py_tensor_map_type = NULL; static inline DevicePtrInfo getPointer(PyObject *obj, int idx) { DevicePtrInfo ptr_info; ptr_info.dev_ptr = 0; ptr_info.valid = true; if (PyLong_Check(obj)) { ptr_info.dev_ptr = PyLong_AsUnsignedLongLong(obj); return ptr_info; } if (obj == Py_None) { // valid nullptr return ptr_info; } PyObject *ret = PyObject_CallMethodNoArgs(obj, data_ptr_str); if (!ret) { PyErr_SetString(PyExc_TypeError, "Pointer argument must be either uint64 or have data_ptr method"); ptr_info.valid = false; goto cleanup; } if (!PyLong_Check(ret)) { PyErr_SetString(PyExc_TypeError, "data_ptr method of Pointer object must return 64-bit int"); ptr_info.valid = false; goto cleanup; } ptr_info.dev_ptr = PyLong_AsUnsignedLongLong(ret); if(!ptr_info.dev_ptr) return ptr_info; uint64_t dev_ptr; int status = cuPointerGetAttribute(&dev_ptr, CU_POINTER_ATTRIBUTE_DEVICE_POINTER, ptr_info.dev_ptr); if (status == CUDA_ERROR_INVALID_VALUE) { PyErr_Format(PyExc_ValueError, "Pointer argument (at %d) cannot be accessed from Triton (cpu tensor?)", idx); ptr_info.valid = false; } else if (status != CUDA_SUCCESS) { CUDA_CHECK(status); // Catch any other cuda API errors ptr_info.valid = false; } ptr_info.dev_ptr = dev_ptr; cleanup: Py_XDECREF(ret); return ptr_info; } static inline CUtensorMap* getTmaDesc(PyObject *obj) { if (sizeof(CUtensorMap*) != 8) { PyErr_SetString(PyExc_SystemError, "getTmaDesc() requires 64-bit compilation"); return NULL; } if (Py_TYPE(obj) != (PyTypeObject*)py_tensor_map_type) { PyErr_Format(PyExc_TypeError, "object must be of type PyCUtensorMap, got %s", Py_TYPE(obj)->tp_name); return NULL; } CUtensorMap* map = &((PyCUtensorMapObject*)obj)->tensorMap; uintptr_t align_128 = (uintptr_t)map & (128 - 1); if (align_128 != 0) { PyErr_Format(PyExc_ValueError, "CUtensorMap must be aligned to 128B, but got (&map) mod 128 = %ld", align_128); return NULL; } return map; } static void ensureCudaContext() { CUcontext pctx; CUDA_CHECK(cuCtxGetCurrent(&pctx)); if (!pctx) { // Ensure device context. CUdevice device; CUDA_CHECK(cuDeviceGet(&device, 0)); CUDA_CHECK(cuDevicePrimaryCtxRetain(&pctx, device)); CUDA_CHECK(cuCtxSetCurrent(pctx)); } } static uint16_t pack_fp16(double f) { uint16_t result; // from https://github.com/python/pythoncapi-compat #if 0x030600B1 <= PY_VERSION_HEX && PY_VERSION_HEX <= 0x030B00A1 && !defined(PYPY_VERSION) _PyFloat_Pack2(f, (unsigned char*)&result, 1); #else PyFloat_Pack2(f, (unsigned char*)&result, 1); #endif return result; } static uint16_t pack_bf16(double f) { float f32 = (float)f; uint32_t u32 = *(uint32_t*)&f32; return (uint16_t)(u32 >> 16); } static uint32_t pack_fp32(double f) { float f32 = (float)f; return *(uint32_t*)&f32; } static uint64_t pack_fp64(double f) { return *(uint64_t*)&f; } static PyObject* launch(PyObject* self, PyObject* args) { // ensure cuda context is valid before calling any CUDA APIs, e.g. before getPointer calls cuPointerGetAttributes ensureCudaContext(); int gridX, gridY, gridZ; uint64_t _stream; uint64_t _function; int launch_cooperative_grid; int launch_pdl; PyObject *launch_enter_hook = NULL; PyObject *launch_exit_hook = NULL; PyObject *kernel_metadata = NULL; PyObject *launch_metadata = NULL; PyObject *global_scratch_obj = NULL; PyObject *profile_scratch_obj = NULL; c8g|]\}}|d|dS)r;r)rrryrs rrz!make_launcher..s8RRRuq"OOB''22a222RRRrz if(!PyArg_ParseTuple(args, "aM", &gridX, &gridY, &gridZ, &_stream, &_function, &launch_cooperative_grid, &launch_pdl, &global_scratch_obj, &profile_scratch_obj, &kernel_metadata, &launch_metadata, &launch_enter_hook, &launch_exit_hookaT)) { return NULL; } int num_warps, num_ctas, shared_memory, clusterDimX, clusterDimY, clusterDimZ; if (!PyArg_ParseTuple(kernel_metadata, "iiiiii", &num_warps, &num_ctas, &shared_memory, &clusterDimX, &clusterDimY, &clusterDimZ)) { PyErr_SetString(PyExc_TypeError, "kernel_metadata must be a tuple"); return NULL; } // extract launch metadata if (launch_enter_hook != Py_None){ PyObject* ret = PyObject_CallOneArg(launch_enter_hook, launch_metadata); if (!ret) return NULL; Py_DECREF(ret); } CUdeviceptr global_scratch = 0; if (global_scratch_obj != Py_None) { DevicePtrInfo global_scratch_info = getPointer(global_scratch_obj, -1); if (!global_scratch_info.valid) { return NULL; } global_scratch = global_scratch_info.dev_ptr; } CUdeviceptr profile_scratch = 0; if (profile_scratch_obj != Py_None) { DevicePtrInfo profile_scratch_info = getPointer(profile_scratch_obj, -1); if (!profile_scratch_info.valid) { return NULL; } profile_scratch = profile_scratch_info.dev_ptr; } // raise exception asap z Py_BEGIN_ALLOW_THREADS; _launch(gridX, gridY, gridZ, num_warps, num_ctas, launch_cooperative_grid, launch_pdl, clusterDimX, clusterDimY, clusterDimZ, shared_memory, (CUstream)_stream, (CUfunction)_function, global_scratch, profile_scratchap); Py_END_ALLOW_THREADS; if (PyErr_Occurred()) { return NULL; } if(launch_exit_hook != Py_None){ PyObject* ret = PyObject_CallOneArg(launch_exit_hook, launch_metadata); if (!ret) return NULL; Py_DECREF(ret); } Py_RETURN_NONE; } static PyMethodDef ModuleMethods[] = { {"launch", launch, METH_VARARGS, "Entry point for all kernels with this signature"}, {NULL, NULL, 0, NULL} // sentinel }; static struct PyModuleDef ModuleDef = { PyModuleDef_HEAD_INIT, "__triton_launcher", NULL, //documentation -1, //size ModuleMethods }; PyMODINIT_FUNC PyInit___triton_launcher(void) { data_ptr_str = PyUnicode_InternFromString("data_ptr"); if(data_ptr_str == NULL) { return NULL; } PyObject* driver_mod = PyImport_ImportModule("triton.backends.nvidia.driver"); if (driver_mod == NULL) { return NULL; } py_tensor_map_type = PyObject_GetAttrString(driver_mod, "PyCUtensorMap"); if (py_tensor_map_type == NULL) { return NULL; } PyObject *m = PyModule_Create(&ModuleDef); if(m == NULL) { return NULL; } PyModule_AddFunctions(m, ModuleMethods); return m; } ) values enumerater"_BASE_ARGS_FORMATritemsrrrzr) constantsrrrexpand_signature args_formatformatflat_signaturer args_list arg_decl_listrry arg_declsinternal_args_listparamsnewline ptr_decls tma_declsfloat_storage_declsrGrrrs ` @@@r make_launcherrs%%%%%N.)()9)9););<<>>),<"="=>>>I''FFFF93C3C3E3EFFFGGK  ,FN!!003////<<)N";";<<> " "FG__&&I enetetevevI__&& N Mioo&7&7 M M MF MM#$$$ MM$%%%N jx{|ExFxFIJxJxJdhktdtdtPRkN N lyy((mN N ^ <<RRRR @Q@QRRRSS_N N `!'aN N fR[gN N p << qN N r << sN N t <<#$$uN N xBEFXBYBY\]B]B]Z^aeajajk}a~a~Z~Z~ceyN N N C^ Jrc#K|]}||fV dSr8r)rrs rr&r&[s&::1A::::::r  c |/|jg|j|j|jdk|j|jS|d}|d}|d}|d}|d}|j}|j}|ddksJ|jdkrdnd } |rt |}|dxxd zcc<t jjjj |j ||t||||| } | g||S) Nnanswizzle elem_size elem_type block_size fp4_paddedrrrr) baserstridespaddinglisttritonruntimedriveractiveutilsrQdata_ptrTMA_DTYPE_DEVICE_TO_HOST) argmetadatarrrrrrrr cu_tensor_maps rmake_tensordesc_argras1c39cs{cCK54Hc39cWZWbccy!G%I%I,'J,'J IEkG 2;!    ;%''aaQGU  b Q N)06JJ  +   M  ,E ,G ,,rcZtd|D}|sStdt|Dr"t t ksJsdgt zfd}|S)Nc3jK|].}t|to|dV/dS)r]Nrr.rx)rrs rr&z)wrap_handle_tensordesc..s>rrX[jc22Ss~~l7S7Srrrrrrrclg|]1\}}t|t|d/|2S)r]r)rrrs rrz*wrap_handle_tensordesc..s@pppvq#*S#:N:NpSVSaSabnSoSopppprct|dt}d}t|tdD]M\}}|vr/|t |||dz }8||N|S)Nrr)r_BASE_ARGS_FORMAT_LENrextendrr)args final_argsrrrlaunchertensordesc_indicesrs rinnerz%wrap_handle_tensordesc..inners$5 55677 %:%;%; <== ' 'FAs&&&!!"5c?>;Z"["[\\\!#!!#&&&&x$$r)r/rsetrr)rrrhas_tensor_desc_argrrs` ` @rwrap_handle_tensordescrsrr_h_o_o_q_qrrrrr pp9#3#3#5#566ppprrQ#o"6"6#>P:Q:Q"Q"Q"Q"Q ;&3'9#:#:: % % % % % % % LrceZdZdZdZdS) CudaLauncherctdrjn t}fdfd|D}djD}t |dd}t |||tdttt}tj tj|jd|_t#|j|||_|j|_|j|_|j|_|j|_|j|_|j|_dS) Nrcrt|tr jj|fn|Sr8)rr.fn arg_namesindex)rrGs rz'CudaLauncher.__init__..s2Z3=O=OVSV-33A6699UVrc.i|]\}}||Srr)ridxvaluearg_idxs rrz)CudaLauncher.__init__..s'MMMZS%WWS\\5MMMrci|]\}}|| Srr)rrrs rrz)CudaLauncher.__init__..sHHHJCS%HHHrr__triton_launcherrFr)r?rdictrrgetattrrrr:rIrJ functoolsreduceoperatormul cluster_dimsnum_ctasrlaunchglobal_scratch_sizeglobal_scratch_alignprofile_scratch_sizeprofile_scratch_alignlaunch_cooperative_grid launch_pdl)rRrGrrrrrSrs ` @rrTzCudaLauncher.__init__s5%,S+%>%>JCMMDFF VVVVMMMM9??;L;LMMM HH#-2E2E2G2GHHH !(,=tDDIy/BB%$%%    "(x7LaPP ,SZOTT #+#? $,$A!$,$A!%-%C"'/'G$"-rc fd}|jjtj}|jjtj} j|jj || g |RdS)Nc|dkr6zz}| jz|z}|}||| SdSNr)rget) sizealign allocator grid_size alloc_sizealloc_fngridXgridYgridZrRstreams rallocate_scratchz/CudaLauncher.__call__..allocate_scratchsQaxx!EME1 &6= $==??x E6:::4r) rrr _allocatorr r!_profile_allocatorrr"r#) rRr.r/r0r1functionrr2global_scratchprofile_scratchs ````` r__call__zCudaLauncher.__call__s         *)$*BDD]_j_uvv**4+DdF`+6+IKK E5%4;WY]Yh"O <6: < < < < < >>rcddl}|jSr&)rFr rGs rget_device_interfacezCudaDriver.get_device_interfaces zrc| ddl}|jo |jjduS#t $rYdSwxYw)NrF)rFr is_availableversionhip ImportError)rFs r is_activezCudaDriver.is_activesU  LLL:**,,L%-2Ct2K L   55 s *- ;;ryreturnc t|Sr8)rz)rRrys rmap_python_to_cpp_typez!CudaDriver.map_python_to_cpp_types}}rcddlm}|S)Nr)do_bench)triton.testingrU)rRrUs rget_benchmarkerzCudaDriver.get_benchmarkers++++++rchddl}d}|t|dz|jdS)Nrir )rrA)rFemptyint)rRrF cache_sizes rget_empty_cache_for_benchmarkz(CudaDriver.get_empty_cache_for_benchmarks8 ' {{3zQ//uy{PPPrc.|dSr8)zero_)rRcaches r clear_cachezCudaDriver.clear_caches r)rUrVrWrTrDrHrJ staticmethodrPr.rSrWr]rarXrYs@rr:r:s 888???\QQQrr:)-rrrr)rrpathlibrrtriton.runtime.buildrtriton.runtimertriton.backends.compilerrtriton.backends.driverrrrrealpath__file__r"rIr9rJrL lru_cacher6r:objectr<rzrrrrrrrrrrrrr:rrrrls  888888&&&&&&......,,,,,, '//"'**844 5 5 Wi001  We,, H  .,,,;;;;;;;;:   4            $-..WWWv 4::b :::::  $-$-$-N2(<(<(<(<(<6(<(<(