Team Ai
Datasetpublic

Brunobkr/llama.cpp_AlgMor24_github

ΩFFFΣLLIa • llama.cpp • AlgMor24 ██████╗ ███████╗███████╗███████╗██╗ ██╗ ██╗ █████╗ ██╔═══██╗██╔════╝██╔════╝██╔════╝██║ ██║ ██║██╔══██╗ ██║ ██║█████╗ █████╗ █████╗ ██║ ██║ ██║███████║ ██║ ██║██╔══╝ ██╔══╝ ██╔══╝ ██║ ██║ ██║██╔══██║ ╚██████╔╝██║ ██║ ███████╗███████╗███████╗██║██║ ██║ ╚═════╝ ╚═╝ ╚═╝ ╚══════╝╚══════╝╚══════╝╚═╝╚═╝ ╚═╝ High-Performance LLM / VLM Inference & Autonomous Agentic Ecosystem… See the full description on the dataset page: https://huggingface.co/datasets/Brunobkr/llama.cpp_AlgMor24_github.

sourceHugging Faceupdated 2mo agoView on Hugging Face
0likes3.1kdownloads
qwen.cpython-313.pyc176 linesDownload Raw Back to __pycache__
1�

2�pzj�}��~�SSKJr SSKrSSKJrJrJrJr SSKr\(aSSKJ	r	 SSK3JrJrJ
r
Jr \R"S5"SS	\55r\R"S4SSS
SS5"SS5\55r\R"S5"SS\55r\R"SS5"SS\55r\R"S5"SS\55r"SS5r\R"S5"SS\\55r\R"S5"S S!\55r"S"S#\5r"S$S%5r\R"S&S'5"S(S)\\55r\R"S*S+5"S,S-\\55r\R"S.5"S/S0\55r\R"S15"S2S3\55rg)4�)�annotationsN)�Any�Callable�Iterable�
TYPE_CHECKING)�Tensor�)�	ModelBase�	TextModel�gguf�logger�QWenLMHeadModel��n�\rSrSr\R6Rr\S5r	\SSSjj5r7SrSrg)	�	QwenModel��	��SSKJn U"5nSRURS5Vs/sHo2[	U5PM sn5$s snf)Nr)�bytes_to_unicode��latin-1)�#transformers.convert_slow_tokenizerr�join�decode�ord)�br�byte_encoder�chars    �6/home/userk21/llama.cpp_algmor24_v2/conversion/qwen.py�token_bytes_to_string�QwenModel.token_bytes_to_strings@��H�'�)���w�w�A�H�H�Y�<O�P�<O�D�S��Y�/�<O�P�Q�Q��Ps�A	N��N�UVs/sHn[U/5PM nnSnSn[[USSUSS55H2upxURUSUS-5n	U	cM$UbX�:dM.UnU	nM4 UbUbXb:�aU$UceUSUXEXES--/-XES-S-nM�s snf)N�����r	r�)�bytes�	enumerate�zip�get)8�mergeable_ranks�token�max_rankr�parts�min_idx�min_rank�i�pair�ranks9          r�bpe�
QwenModel.bpes���%*�+�U����s��U��+���G��H�$�S��s���U�1�2�Y�%?�@���&�*�*�4��7�T�!�W�+<�=���#��)9�T�_��G�#�H�	A�10��H�$8�X�=Q�����&�&�&��(�7�O�u�~���k�8J�'J�&K�K�e�^_�T_�T`�Na�a�E���,s�B"��$�UR5 g�N)�_set_vocab_qwen��selfs r�	set_vocab�QwenModel.set_vocab*s�������r7)r*�dict[bytes, int]r+r&r,�11int | None�return�list[bytes])
�__name__�12__module__�__qualname__�__firstlineno__r�13MODEL_ARCH�QWEN�14model_arch�staticmethodr r3r;�__static_attributes__r>r=rrrs>�����%�%�J��R��R�15���� r=r�16Qwen2Model�Qwen2ForCausalLM�"Qwen2AudioForConditionalGeneration�KORMoForCausalLM�&AudioFlamingo3ForConditionalGeneration�DotsOCRForCausalLM��j^�\rSrSr\R17RrSrU4Sjr	SU4Sjjr18SrU=r$)rL�.��f�UR5 g![a UR5 gf=fr7)�_set_vocab_sentencepiece�FileNotFoundError�_set_vocab_gpt2r9s rr;�Qwen2Model.set_vocab9s.��	#��)�)�+�� �	#�� � �"�	#�s��0�0��B>�[TU]5 UR5 gr7)�super�set_gguf_parameters�_try_set_pooling_type�r:�	__class__s �rr^�Qwen2Model.set_gguf_parameters?s���
��#�%��"�"�$r=�#�l># �URS:XaSU3n[TU]	XU5Shv�N gN7f)NrL�model.)�hf_archr]�modify_tensors)r:�19data_torch�name�bidras    �rrg�Qwen2Model.modify_tensorsCs2�����<�<�<�'��D�6�?�D��7�)�*�C�@�@�@�s�)4�2�4r>�rhrri�strrjr@rA�Iterable[tuple[str, Tensor]])
rCrDrErFrrG�QWEN2rIr;r^rgrK�
__classcell__�ras@rrLrL.s*������&�&�J�#�%�A�Ar=�Qwen2MoeForCausalLM��^�\rSrSr%\R20RrU4SjrSr	S\21S'S	U4SjjrU4SjrSr
U=r$)22�
Qwen2MoeModel�I��l>�[TU]5 URRS5=nb4URRU5 [R"SU35 URRS5=nb5URRU5 [R"SU35 gg)N�moe_intermediate_size�#gguf: expert feed forward length = �shared_expert_intermediate_size�*gguf: expert shared feed forward length = )	r]r^�hparamsr)�gguf_writer�add_expert_feed_forward_lengthr
�info�%add_expert_shared_feed_forward_length)r:ryr{ras   �rr^�!Qwen2MoeModel.set_gguf_parametersMs����
��#�%�%)�\�\�%5�%5�6M�%N�N�!�[����;�;�<Q�R��K�K�=�>S�=T�U�V�/3�|�|�/?�/?�@a�/b�b�+�o����B�B�Cb�c��K�K�D�Ed�De�f�g�pr=N�list[dict[str, Tensor]] | None�_experts�	#�># �URS5(dURS5(a6URS5(dUS3OUn[TU]	XU5Shv�N gURS5(dURS5(a�URS:dURSS-S	:wa$[S23US[
UR535eURSS-nUSSU2SS24R5nUSUS2SS24R5nURS5RS
5nUS3n	US3n24[TU]	XiU5Shv�N [TU]	XzU5Shv�N gURS5S:wGaURSS/5nUceURc*[UR5Vs/sHn0PM snUlXRUU'[URU5US-:�a�SH�n
/n[U5H@nSUSUSU
S3nURURUU5 URUU	MB [ R""US	S9nSUSU
S3n[TU]	UUU5Shv�N M� gg[TU]	XU5Shv�N gGN^GNbGNMs snfN1N7f)N�mlp.experts.down_proj�mlp.experts.down_proj.weight�.weight�mlp.experts.gate_up_proj�mlp.experts.gate_up_proj.weight������r%r�"Unexpected gate_up_proj shape for �: .�
.gate_up_proj�.gate_proj.weight�.up_proj.weight�expertsr$�num_local_experts�num_experts)�	down_proj�	gate_proj�up_proj�
model.layers.�
.mlp.experts.�.��dim)�endswithr]rg�ndim�shape�25ValueError�tuple�26contiguous�removesuffix�find�find_hparamr��range�block_count�len�append�torch�stack)r:rhrirj�mapped�n_ff�gate�up�	base_name�mapped_gate�	mapped_up�	n_experts�_�w_name�datas�xid�ename�merged_nameras                  �rrg�Qwen2MoeModel.modify_tensorsXs������=�=�0�1�1�T�]�]�Ca�5b�5b�-1�]�]�9�-E�-E��v�W�%�4�F��w�-�j�#�F�F�F���=�=�3�4�4��
�
�Fg�8h�8h�����"�j�&6�&6�r�&:�Q�&>�!�&C� �#E�d�V�2�e�T^�Td�Td�Ne�Mf�!g�h�h��#�#�B�'�1�,�D��c�5�D�5�!�m�,�7�7�9�D��C����M�*�5�5�7�B��)�)�)�4�A�A�/�R�I�&�K�'8�9�K�$�+�_�5�I��w�-�d��E�E�E��w�-�b�S�A�A�A���9�9�Y��2�%��(�(�*=�}�)M�N�I��?�"�?��}�}�$�-2�4�3C�3C�-D� E�-D���-D� E��
�'1�M�M�#��t�$��4�=�=��%�&�)�a�-�7�C�F�*,�E�$�Y�/��"/��u�M�#��a��x�w� W�����T�]�]�3�%7��%>�?� �M�M�#�.�u�5� 0�27"'���U��!:�J�$1�#��m�F�8�7�"S�K�$�w�5�j�+�s�S�S�S�D����7�)�*�C�@�@�@�]
G�
F�A��!F�$T�28	A�si�AK� J>�!C>K�K� K�7K�8AK�K�CK�K�K�8K�9K�K�K�K�K���>�[TU]5 URbPURVVs/sHoR5Ho"PM M nnn[	U5S:�a[SU35eggs snnf)Nr�Unprocessed experts: )r]�prepare_tensorsr��keysr�r�)r:�d�kr�ras    �rr��Qwen2MoeModel.prepare_tensors�sh���
���!��=�=�$�"&�-�-�B�-�Q����A�q��q�-�G�B��7�|�a�� �#8��	�!B�C�C� �%��Bs� A.)r�rl)rCrDrErFrrG�QWEN2MOErIr^r��__annotations__rgr�rKrprqs@rruruIs8������)�)�J�h�04�H�,�3�7A�rD�Dr=ru�Qwen3ForCausalLM�29Qwen3Model���^�\rSrSr%\R30RrSrS\	S'Sr31S\	S'SrS\	S'SrS\	S	'U4S32jr
SSjrU4SjrS
rU4SjrSSjrSU4SjjrSrU=r$)r��F�bool�	is_rerank�is_tied_embeddingsNr@�token_false_id�
token_true_id���>�[TU]"U0UD6 [R"URSS9nURSS/5SUlUR5(aUR5 gg)NF)�is_mistral_format�
architecturesr)	r]�__init__r33�load_hparams�	dir_modelr)�origin_hf_arch�_is_qwen3_reranker�_find_rerank_config�r:�args�kwargsr}ras    �rr��Qwen3Model.__init__�sg���
���$�)�&�)��(�(����5�Q��%�k�k�/�D�6�B�1�E����"�"�$�$��$�$�&�%r=����URS-nSnUR5(a)URSSS9nUR5nSSS5 [	URR345[	URRSS55[	URRSS55[	UR=(d S5/nUVs/sHoU(dMUR5PM nnSUR5;dS	UR5;ag35[SU55(ag36SUR=(d SR5;$!,(df   GN"=fs snf)
N�	README.mdr�r�utf-8��encoding�
_name_or_path�37model_type�# qwen3-reranker�# qwen3-vl-rerankerT�3�B# �UHnSU;=(d SU;v� M g7f)�qwen3-reranker�qwen3-vl-rerankerNr>)�.0�hints  r�	<genexpr>�0Qwen3Model._is_qwen3_reranker.<locals>.<genexpr>�s&���^�S]�4��4�'�F�+>�$�+F�F�S]�s��sequenceclassification)r��exists�open�readrmrir}r)r��lower�any)r:�readme_path�readme_text�f�38name_hintsr�s      rr��Qwen3Model._is_qwen3_reranker�s1���n�n�{�2����������!�!�#��!�8�A��f�f�h��9�
����#�#�$����� � ��"�5�6����� � ��r�2�3���#�#�)�r�*�	39�40�0:�B�z�t�T�l�d�j�j�l�z�41�B���!2�!2�!4�4�8M�Q\�Qb�Qb�Qd�8d���^�S]�^�^�^��'�D�,?�,?�,E�2�+L�+L�+N�N�N�#9�8��Cs�E#�	E5�&E5�#42E2��d>�URS:XaUR5 g[TU]
5 g�N� InternS1ForConditionalGeneration�r��_set_vocab_interns1r]r;r`s �rr;�Qwen3Model.set_vocab��,������"D�D��$�$�&��
���r=��R�SSKJn URUR5nSUlUR43R
SS5UlURS5Ul	URS5Ul44URS5UlURb
URceg)	Nr)�
AutoTokenizerT�tie_word_embeddingsF�no�yes�|)�transformersr�from_pretrainedr�r�r}r)r��convert_tokens_to_idsr�r��sep_token_id)r:r�	tokenizers   rr��Qwen3Model._find_rerank_config�s���.�!�1�1�$�.�.�A�	����"&�,�,�"2�"2�3H�%�"P���'�=�=�d�C���&�<�<�U�C���%�;�;�C�@����"�"�.�4�3E�3E�3Q�Q�Q�3Qr=��$>�[TU]5 UR(apURR	[45RR5 URRSS/5 URRSSS./5 gg)Nr	r�rerank�p<|im_start|>system46Judge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be "yes" or "no".<|im_end|>47<|im_start|>user48<Instruct>: Given a web search query, retrieve relevant passages that answer the query49<Query>: {query}50<Document>: {document}<|im_end|>51<|im_start|>assistant52<think>53 54</think>55 56)ri�template)57r]r^r�r~�add_pooling_typer�PoolingType�RANK�add_classifier_output_labels�add_chat_templater`s �rr^�Qwen3Model.set_gguf_parameters�sy���
��#�%��>�>����-�-�d�.>�.>�.C�.C�D����9�9�5�$�-�H����.�.� �M�1�0�
�r=��f�XRnXRn[R"X2/SS9$)Nrr�)r�r�r�r�)r:rh�	false_row�true_rows    r�_get_cls_out_tensor�Qwen3Model._get_cls_out_tensor�s2���2�2�3�	��0�0�1���{�{�H�0�a�8�8r=�#�># �UR(a�UR=(a SU;nUR(+=(a SU;nU(dU(ac[R[RR58S-UR
U54nUv� U(a[TU]!XU5Shv�N g[TU]!XU5Shv�N gNN7f)N�embed_tokens�lm_headr�)	r�r�r�TENSOR_NAMES�MODEL_TENSOR�CLS_OUTr r]rg)r:rhrirj�is_tied_head�is_real_head�cls_out_headras       �rrg�Qwen3Model.modify_tensors�s������>�>��2�2�M�~��7M�L�#�6�6�6�L�9��;L�L��|��%�%�d�&7�&7�&?�&?�@�9�L��,�,�Z�8� ��#�"��$�w�5�j��L�L�L���7�)�*�C�@�@�@�M�	A�s$�B2C�5C�6C�C�C�C)r�r�r�rr�r�)rAr�)rhrrArrl)rCrDrErFrrG�QWEN3rIr�r�r�r�r�r�r�r;r�r^r rgrKrprqs@rr�r��si������&�&�J��I�t��$���$�!%�N�J�%� $�M�:�$�'�O�.�59R�60�9�A�Ar=�Qwen3MoeForCausalLM��`^�\rSrSr\R61RrU4SjrU4Sjr	Sr62U=r$)�
Qwen3MoeModel����>�[TU]"U0UD6 [R"URS5nURSS/5SUlg)NFr�r)r]r�r63r�r�r)r�r�s    �rr��Qwen3MoeModel.__init__sE���
���$�)�&�)��(�(�����?��%�k�k�/�D�6�B�1�E��r=��d>�URS:XaUR5 g[TU]
5 gr�rr`s �rr;�Qwen3MoeModel.set_vocabrr=)r�)rCrDrErFrrG�QWEN3MOErIr�r;rKrprqs@rr1r1�s"������)�)�J�F�64�r=r1���^�\rSrSr%SrSrS\S'S\S'S\S	'S65\S'S\S
'S\S'S\S'SrS\S'SrS66\S'U4Sjr	SSU4Sjjjr67\S5rU4Sjr
SU4SjjrSrU=r$)�
_QwenMtpMixini�vShared MTP wiring for Qwen3-Next and Qwen3.5/3.6 text variants. The HF68config carries the MTP block under `mtp_num_hidden_layers` (computed from69the checkpoint when absent, e.g. Qwen3-Next) and the tensors under70`mtp.*`; we extend block_count, emit the nextn metadata key, and remap71`mtp.*` to the standard layer-indexed nextn naming so the existing72tensor_map handles them.T�dict[str, Any]r}�gguf.MODEL_ARCHrI�gguf.GGUFWriterr~�intr��gguf.TensorNameMap�73tensor_mapr��no_mtp�mtp_onlyNr@�_original_block_countr�opt_num_mtp_layers��z>�[TU]"U0UD6 URSUlUR(dUURRSS5nUS:XaURS:wdeURnU=RU-
sl[R"URUR5Ul74g)N�num_hidden_layers�mtp_num_hidden_layersr)r]r�r}r�rDr)rGr�get_tensor_name_maprIrC)r:r�r��n_mtpras    �rr��_QwenMtpMixin.__init__"s����
���$�)�&�)��<�<�(;�<����{�{��L�L�$�$�%<�a�@�E���z��.�.�!�3�3�3��/�/������%���2�2�4�?�?�D�DT�DT�U��r=���>^�0UREURRS05Em[U4SjS5S5nTRU5[U5lS[U5l[TU]US9$)N�text_config�3�4># �UHoT;dM75Uv� M g7fr7r>)r�r�r}s  �rr��._QwenMtpMixin.index_tensors.<locals>.<genexpr>0s����j�Y�!�bi�]i�A�A�Y�s��	)�n_layersrJ�n_layer�76num_layersr)�remote_hf_model_id)r}r)�next�typerFrGr]�
index_tensors)r:rX�keyr}ras   @�rr[�_QwenMtpMixin.index_tensors.sp���I�T�\�\�I�T�\�\�%5�%5�m�R�%H�I���j�Y�j�lp�q��+2�;�;�s�+;��T�77�(�()��T�78�%��w�$�8J�$�K�Kr=����URce[R"U5=ncgUup4URS5(aUR	SSS5nURS5(a�UR79(agSSSS	S80.nUR
SS5n[U5S
:XahUSS:Xa_USR5(aG[US5nSURU-SUS3n[URUS-5UlX44$[U5S:Xa&USU;aSURSXVSSUS3nX44$UR(a
US;nU(dgX44$)N�81model.mtp.rerr	�mtp.�eh_proj�enorm�hnorm�shared_head.norm)�fc�pre_fc_norm_embedding�pre_fc_norm_hidden�normr�r���layersr%r�)�model.embed_tokens.weight�model.norm.weight�lm_head.weight�embed_tokens.weight�norm.weight)
rFr�filter_tensors�82startswith�replacerD�splitr��	isdecimalrA�maxrGrE)	�cls�item�titemri�gen�remapperr-�mtp_idx�keeps	         rrq�_QwenMtpMixin.filter_tensors5s����(�(�4�4�4��-�-�d�3�3�E�<���	���?�?�<�(�(��<�<��"�a�0�D��?�?�6�"�"��z�z��)2�)0�)0�);�	�H��J�J�s�A�&�E��5�z�Q��5��8�x�#7�E�!�H�<N�<N�<P�<P��e�A�h�-��&�s�'@�'@�7�'J�&K�1�U�ST�X�J�W��),�S�-C�-C�W�q�[�)Q��&��y���U��q��U�1�X��%9�&�s�'@�'@�&A��8�RS�H�CU�BV�VW�X]�^_�X`�Wa�b���y���\�\����D����y�r=���>�[TU]5 UR(agURURS-83=nS:�aUR84R
U5 gg)NrJr)r]r^rDr�r}r~�add_nextn_predict_layers)r:�nras  �rr^�!_QwenMtpMixin.set_gguf_parametersWsT���
��#�%��;�;���!�!�D�L�L�1D�$E�E�E�A��J����5�5�a�8�Kr=�85��>�URR5n[TU]
US9 UR(aU(dgUR86RRS5Sn[R"URRURRURRURRSUSS9nURRSUS3-Ulg)N)�87vocab_onlyr�r%)�88size_label�output_typer��mtp-�.gguf)�	fname_out�is_dirr]�prepare_metadatarE�ftyperi�	partitionr�naming_convention�metadata�basename�finetune�version�parent)r:r��from_dirr��
fname_defaultras     �rr��_QwenMtpMixin.prepare_metadata^s�����>�>�(�(�*��
�� �J� �7��}�}�H���:�:�?�?�4�4�S�9�!�<��!�3�3��M�M����
�
� 6� 6��
�
�8N�8N��M�M�!�!�d��X\�^�
����.�.�4�
��e�1L�L��r=)r�r�rCr7)rX�89str | NonerA�dict[str, Callable[[], Tensor]])r�r�)rCrDrErF�__doc__�supports_mtp_exportr�rFrGr�r[�classmethodrqr^r�rKrprqs@rr<r<s���� ���
���� � ���"�"��L��N�(,��:�,�����90V�L�L�����B9�M�Mr=r<�Qwen3NextForCausalLM��d^�\rSrSr\R91RrU4SjrSU4Sjjr	Sr92U=r$)�Qwen3NextModelil�	�6>�[TU]5 URRURS5 URRURS5 URR
URS5 URRURS5 URRURSURS-5 URRURRSS55 URRS5=ncURS	URS93-nURR[XRRSS5-55 g)
N�linear_conv_kernel_dim�linear_key_head_dim�linear_num_key_heads�linear_num_value_heads�linear_value_head_dim�full_attention_intervalrj�head_dim�hidden_size�num_attention_heads�partial_rotary_factorg�?)r]r^r~�add_ssm_conv_kernelr}�add_ssm_state_size�add_ssm_group_count�add_ssm_time_step_rank�add_ssm_inner_size�add_full_attention_intervalr)�add_rope_dimension_countrA�rope_parameters)r:�rope_dimras  �rr^�"Qwen3NextModel.set_gguf_parameterspsE���
��#�%����,�,�T�\�\�:R�-S�T����+�+�D�L�L�9N�,O�P����,�,�T�\�\�:P�-Q�R����/�/����=U�0V�W����+�+�D�L�L�9P�,Q�TX�T`�T`�ay�Tz�,z�{����4�4�T�\�\�5E�5E�F_�ab�5c�d����(�(��4�4�H�=��|�|�M�2�d�l�l�CX�6Y�Y�H����1�1�#�h�AU�AU�AY�AY�Zq�sw�Ax�6x�2y�zr=�#�^># �URS5(a[R"U5*nOvURS5(aURS5SS-nOHSU;aUR	5nO1URS5(aURS5(dUS-nS	U;Ga�UR94S95nUR96SnUR97SnUR98S
nUR99SnUUXg-U-Xg-U-/n	UR
SS5R5nURSU[U	55n[R"XSS9up�p�U100R5RUS5n101UR5RUS5nUR5RUS5nU
R5RUS5n
[R"X�U/SS9R
SS5R5nU
R
SS5R5n
UR[RRUS5U4v� UR[RR US5U
4v� g["TU]IXU5Shv�N gN7f)N�.A_log�.dt_biasr�
.dt_proj.bias�conv1drp�linear_attn.norm.weightr	�in_proj_qkvz.weightr�r�r�r�r�r$r�r�)r�r��exp�102rpartition�squeezer}�permuter��view�sumrt�cat�format_tensor_namerr'�ATTN_QKV�	ATTN_GATEr]rg)r:rhrirj�103head_k_dim�104head_v_dim�num_v_heads�num_k_headsr��split_arg_list_qkvz�qr��v�z�qkvras               �rrg�Qwen3NextModel.modify_tensors|sP�����=�=��"�"��)�)�J�/�/�J�
�]�]�:�
&�
&��?�?�:�.�q�1�O�C�D�
��
�#�+�+�-�J�
�]�]�=�
)�
)�$�-�-�@Y�2Z�2Z�#�a��J� �D�(����&;�<�J����&=�>�J��,�,�'?�@�K��,�,�'=�>�K��,�,�}�5�K����+�j�8��+�j�8�	#��$�+�+�A�q�1�<�<�>�J�#����[�#�>Q�:R�S�J����Z�"�M�J�A�!�����#�#�K��4�A�����#�#�K��4�A�����#�#�K��4�A�����#�#�K��4�A��)�)�Q�1�I�2�.�6�6�q�!�<�G�G�I�C��	�	�!�Q��*�*�,�A��*�*�4�+<�+<�+E�+E��Y�W�Y\�]�]��*�*�4�+<�+<�+F�+F��Y�W�YZ�[�[��w�-�j��D�D�D�s�J"J-�%J+�&J-r>rl)rCrDrErFrrG�	QWEN3NEXTrIr^rgrKrprqs@rr�r�ls$������*�*�J�105{�(E�(Er=r��RND1��T^�\rSrSr\R106RrU4SjrSr	U=r107$)�	RND1Modeli����>�[TU]5 URRS5 URRS5=nbURR
U5 gg)NF�
mask_token_id)r]r^r~�add_causal_attentionr}r)�add_mask_token_id)r:r�ras  �rr^�RND1Model.set_gguf_parameters�sX���
��#�%�	
���-�-�e�4�!�\�\�-�-�o�>�>�M�K����.�.�}�=�Lr=r>)rCrDrErFrrGr�rIr^rKrprqs@rr�r��s������%�%�J�>�>r=r���^�\rSrSr\R108Rr\SSj5r	SSjr109S	U4SjjrS110U4SjjrSr
U=r$)�_LinearAttentionVReorderBasei���J�[UR5nUS:aU[U5-
nUSUX#U/-XQS-S-nUR"U6n[[	[U555nXqS-XqsXq'XqS-'UR111"U6R
5R"U6$)�RReorder V heads from grouped (by K head) to tiled order along the given dimension.rNr	)�listr�r��reshaper�r�r�)�tensorr�r��num_v_per_kr�r��	new_shape�perms        r�_reorder_v_heads�-_LinearAttentionVReorderBase._reorder_v_heads�s����V�\�\�"����7��3�u�:��C��$�3�K�;�X�"F�F��UV�w�x��X�	�����+���E�#�i�.�)�*��#'�a��=�$�)� ��	�4�a��=��~�~�t�$�/�/�1�9�9�5�A�Ar=��~^^^^^^�URS5(dX#4$TRSmTRSmTRSnTRSnTT-mSSjmSSjmSUU4SjjnSUUUU4S	jjnURS1125(agUT-nUT-n	USUn113X(X�-nX(U	-SnUSUn
X8X�-nX8U	-SnU"X�U5up�[R"X�U/SS9[R"X�U/SS94$URS
5(aU"X#U5up#X#4$URS5(aU"X#S5up#X#4$URS5(abTR	[R114"TU-[RS9RS5STTU5RS5nU"X#U5up#X#4$)N)�.linear_attn.in_proj_qkv.weight�.linear_attn.in_proj_z.weight�.linear_attn.in_proj_a.weight�.linear_attn.in_proj_b.weight�.linear_attn.out_proj.weightr�r�r�r�����[R"US5n[R"US5n[R"X4SS9R"/UR115SSQUR116SS-P76$)Nrrjr$r�r%)r��bitwise_and�bitwise_right_shiftr�r�r�)�qs�lo�his   r�unpack_nibbles�L_LinearAttentionVReorderBase._transform_nvfp4_weight.<locals>.unpack_nibbles�se���"�"�2�t�,�B��*�*�2�q�1�B��;�;��x�R�0�8�8�Z�"�(�(�3�B�-�Z����RT��XY�IY�Z�Zr=��B�UR"/URSSQURSS-PSP76n[R"USS5n[R"[R"USS5S5n[R117"X5R
5$)Nr$r%).rr).r	rj)r�r�r�r��bitwise_left_shift�118bitwise_orr�)�codesr�r�s   r�pack_nibbles�J_LinearAttentionVReorderBase._transform_nvfp4_weight.<locals>.pack_nibbles�s����M�M�M�5�;�;�s��#3�M�U�[�[��_��5I�M�1�M�E��"�"�5��=�$�7�B��)�)�%�*;�*;�E�&�M�4�*P�RS�T�B��#�#�B�+�6�6�8�8r=��>�URS:�deURS:�deURSS-nUR5U:XdeUS-S:XdeURSS5nUSS2S4nUR	S5[119R"SURS9-n[120R"XF5(de[121R"US-S:H5(deUS-R[122RS9n[123R"URS[124RS9nUR5URS:Xde[125R"[126R"U5RU5(deT"U5n	U	RSURUR[127RS95n	T128"U	5nURSURURS95nX4$)	Nr%r$�rr	��dtype)�devicer�r	)r�r��numelr��	unsqueezer��aranger�equal�all�to�long�sort�values�index_selectr	)r��scales�col_permr��129group_cols�group_starts�expected�130group_perm�expected_groupsrrr�s          ��r�apply_col_perm�L_LinearAttentionVReorderBase._transform_nvfp4_weight.<locals>.apply_col_perm�s�����7�7�a�<��<��;�;�!�#�#�#������q� �A��>�>�#�q�(�(�(��r�6�Q�;��;�!�)�)�"�b�1�J�%�a��d�+�L�#�-�-�a�0�5�<�<��(�.�.�3Y�Y�H��;�;�z�4�4�4�4��9�9�\�B�.�!�3�4�4�4�4�&�"�,�0�0�u�z�z�0�B�J�#�l�l�6�<�<��+;�5�:�:�N�O��#�#�%����b�)9�9�9�9��;�;�u�z�z�*�5�<�<�o�N�N�N�N�"�2�&�E��&�&�r�8�;�;�b�i�i�u�z�z�;�+Z�[�E��e�$�B��(�(��Z�]�]�&�-�-�]�-P�Q�F��:�r=��V>�TR[R"TU-[RS9R	S5STTU5RS5nUR
SURURS95UR
SURURS954$)Nrr$rr131)	r�r�r
rrr�rrr	)r�rr��row_permr�r�r�r:s    ����r�reorder_rows�J_LinearAttentionVReorderBase._transform_nvfp4_weight.<locals>.reorder_rowss�����,�,����[�8�3�5�:�:�F�P�P�QS�T��;��X���g�b�k�
�132����8�;�;�b�i�i�;�#@�A��#�#�A�x�{�{�&�-�-�{�'H�I��
r=r�rr�r�)r�r�r	r�r)r�rrAr)rrrAr)r�rrrrrrA�tuple[Tensor, Tensor])r�rrrr�rArAr#)	r�r}r�r�r�r
rrr�)r:ri�weight�scaler�r�rr!�q_dim�k_dimr�r�r��q_scale�k_scale�v_scalerr�r�r�rr�s`                @@@@@r�_transform_nvfp4_weight�4_LinearAttentionVReorderBase._transform_nvfp4_weight�s�����}�}�133����=� ��l�l�#9�:���l�l�#;�<���\�\�"7�8�134��\�\�"9�:�135�!�[�0��	[�136	9�	�	�2	�	��=�=�:�;�;���,�E���,�E��v���A��U�]�+�A��u�}�~�&�A��F�U�m�G��%�-�0�G��E�M�N�+�G�%�a�*�=�J�A��9�9�a�A�Y�A�.��	�	�7�W�:U�[\�0]�]�]��=�=�8�9�9�(��137�C�M�F��}���]�]�]�
^�
^�(���:�M�F��}���]�]�9�
:�
:��,�,����[�:�5�U�Z�Z�H�R�R�ST�U��;��Z���g�a�j�
�+�6�(�C�M�F��}�r=��P>�URXU5up#[TU]	XX4U5 gr7)r+r]�
_repack_nvfp4)r:rir$r%�scale2�input_scaleras      �rr/�*_LinearAttentionVReorderBase._repack_nvfp4&s(����4�4�T�5�I�
��
���d�E�;�Gr=�#��># �URRSS5nURRSS5nUS:�Ga�US:�Ga�XE:wGa�SU;Ga�URSnURSnXT-nSU;aGXd-n	Xd-n138USU	nXX�-nXU139-Sn
URU
SXHU5n
[R"X�U
/SS9nGOS	U;aURUSXHU5nO�S140U;dSU;aURUSXHS5nO�S
U;dSU;dSU;aXUR141S:Xa3URUR
S5SXHS5RS5nO�URUSXHS5nOlSU;aLUR5nXd-S-nUSUnX�SnURUSXHU5n[R"UU/SS9nOSU;aURUSXHU5n[TU]%XU5Shv�N gN7f)Nr�rr��linear_attn.r�r��
.in_proj_qkv.r��.in_proj_z.�.in_proj_b.�.in_proj_a.r	r�r��.dt_projr$�.conv1dr%�142.out_proj.)143r}r)r�r�r�r�rr�r]rg)r:rhrirjr�r�r�r�r�r&r'r�r�r��data�qk_channels�qk_part�v_partras                  �rrg�+_LinearAttentionVReorderBase.modify_tensors*s&�����l�l�&�&�'=�q�A���l�l�&�&�'?��C����?�{�Q��;�3M�R`�dh�Rh����&;�<�J����&=�>�J�%�4�K��$�&�"�0��"�0���v��&���U�]�3���u�}�~�.���)�)�!�Q��*�U��"�Y�Y��a�y�a�8�144��$�&�!�2�2�:�q�+�\f�g�145��$�&�-�4�*?�!�2�2�:�q�+�\]�^�146��T�!�Z�4�%7�:��;M��?�?�a�'�!%�!6�!6�"�,�,�R�0�!�[�q�"��g�b�k��"&�!6�!6�z�2�{�ab�!c�J��d�"�!�)�)�+��(�6��:���|��,���l�+���.�.�v�q�+�T^�_��"�Y�Y���'8�a�@�147���%�!�2�2�:�q�+�\f�g�148��7�)�*�C�@�@�@�s�G(G3�+G1�,G3r>)r�rr�rAr�rAr�rAr�rArAr)rirmr$rr%rrAr#)149rirmr$rr%rr0rr1rrl)rCrDrErFrrGr�rIrJr�r+r/rgrKrprqs@rr�r��sF������*�*�J�	��	B��	B�U�nH�1A�1Ar=r���F^�\rSrSr%/SQrS\S'S\S'U4SjrSrU=r$)	�_Qwen35MRopeMixini^)�rE�150rr@r~�dictr���>�[TU]5 SUR;a&URR	UR1515 gg)N�
mrope_section)r]r^r�r~�add_rope_dimension_sections�_QWEN35_DEFAULT_MROPE_SECTIONr`s �rr^�%_Qwen35MRopeMixin.set_gguf_parametershs;���
��#�%��$�"6�"6�6����8�8��9[�9[�\�7r=r>)	rCrDrErFrLr�r^rKrprqs@rrDrD^s#���152%4�!� � ���]�]r=rD�Qwen3_5ForConditionalGeneration�Qwen3_5ForCausalLM��@�\rSrSr\R153RrSrg)�Qwen3_5TextModelinr>N)	rCrDrErFrrG�QWEN35rIrKr>r=rrRrRns�����'�'�Jr=rR�"Qwen3_5MoeForConditionalGeneration�Qwen3_5MoeForCausalLM��@�\rSrSr\R154RrSrg)�Qwen3_5MoeTextModelisr>N)	rCrDrErFrrG�	QWEN35MOErIrKr>r=rrXrXss�����*�*�Jr=rX�DFlashDraftModel��z^�\rSrSr\R155RrU4SjrU4Sjr	\156SU4Sjj5rSrU=r
$)�DFlashModelix��j>�URc[S5e[R"SUR35 URnURUlSSKJn [URS-SSS9n[R"U5S	S157nSSS5 U"W5nU[U5LaURU5 O[TU]-5 XlURRS05RS5nUbURR!U5 gg!,(df   N�=f)
NڏDFlash draft model requires --target-model-dir to be specified. Please provide the path to the target model directory containing the tokenizer.�+DFlash: Using tokenizer from target model: r	)�get_model_class�config.jsonr�r�r�r�r�
dflash_configr�)�target_model_dirr�r
r�r�rrbr��json�loadrZr;r]r}r)r~r�)r:�original_dirrbr��target_arch�158target_clsr�ras       �rr;�DFlashModel.set_vocab|s���� � �(��b��
�	���A�$�BW�BW�AX�Y�Z��~�~���.�.���	&�
�$�'�'�-�7��w�
O�ST��)�)�A�,��7��:�K�P�$�[�1�159��T�$�Z�'�� � ��&��G���%�����(�(��"�=�A�A�/�R�
��$����.�.�}�=�%�P�
O�s�8D$�$160D2���>�[TU]5 URRSS5nURRU5 URRS05nURS/5nU(a.UVs/sHoDS-PM	 nnURR
U5 URRSS5nURRS5nURRS	5nU(aZU(aRU(aJUV	s/sHo�S161:HPM	 n162n	URRU5 URRU1635 ggggs snfs sn	f)N�164block_sizerrd�target_layer_idsr	�use_sliding_windowF�sliding_window�layer_types�sliding_attention)	r]r^r}r)r~�add_block_size�add_target_layers�add_sliding_window�add_sliding_window_pattern)r:rnrdror0�extract_layer_idsrprqrr�lt�is_swaras           �rr^�DFlashModel.set_gguf_parameters�s2���
��#�%��\�\�%�%�l�B�7�165����'�'�166�3����(�(��"�=�
�(�,�,�-?��D���0@� A�0@�1�Q��0@�� A����.�.�/@�A�!�\�\�-�-�.B�E�J�����)�)�*:�;���l�l�&�&�}�5���.�[�:E�F�+�B�/�/�+�F�F����/�/��?����7�7��?�6A�.���
!B��Gs�
E�
E#��b>�Uup#URS5(dSU-n[TU]	X#45$)Nre)rrr]rq�rwrxrirzras    �rrq�DFlashModel.filter_tensors�s4����	�����x�(�(��d�?�D��w�%�t�k�2�2r=)r��rx� tuple[str, Callable[[], Tensor]]rA�'tuple[str, Callable[[], Tensor]] | None)rCrDrErFrrG�DFLASHrIr;r^r�rqrKrprqs@rr]r]xs1������'�'�J�>�8@�(�3��3r=r]�Qwen3DSparkModel��n^�\rSrSr\R167RrU4Sjr\	SU4Sjj5r168SrU=r$)�DSparkModeli��	��>�[TU]"U0UD6 URRSSVs0sH#o3UR;dMX0RU_M% sn5 gs snf)Nrd)ror�)r]r�r}�169setdefault)r:r�r�r�ras    �rr��DSparkModel.__init__�s`���
���$�)�&�)�������(M�2170�(M�1�VZ�Vb�Vb�Qb��A�|�|�A���(M�2171�	��2172s�A173�A174��Z>�Uup#URS5(ag[TU]	X#45$)N)rorn)r�r]rqr~s    �rrq�DSparkModel.filter_tensors�s/����	���=�=�B�C�C���w�%�t�k�2�2r=r>r�)
rCrDrErFrrGr�rIr�r�rqrKrprqs@rr�r��s-������'�'�J���3��3r=r�)�175__future__rrf�typingrrrrr�r�baser176rrr
�registerrrLrur�r1r<r�r�r�rDrRrXr]r�r>r=r�<module>r�s3��"��9�9����4�4����%�&��	��'��<�����(��,��
�A��A��A�&���)�*�ND�I�ND�+�ND�b���&��5�^A��^A�6�^A�B���)�*��M��+��"ZM�ZM�z���*�+�7E�]�M�7E�,�7E�t���F��>�
�>��>�eA�>�eA�P
]�
]� ���5�7K�L�(�(�*F�(�M�(����8�:Q�R�+�+�-I�+�S�+����&�'�83�*�83�(�83�v���&�'�3�+�3�(�3r=
Brunobkr/llama.cpp_AlgMor24_github · Team Ai