Hugging Face's logo

software-mansion
/
react-native-executorch-llama-3.2

Text Generation
ExecuTorch
conversational
Model card Files
xet
 Community
msluszniak commited on
Commit
4ba07c3
·
verified ·
1 Parent(s): ab3b8e2

Add MLX int4 variants for 1B and 3B

Browse files
Files changed (5) hide show
  1. .gitattributes +2 -0
  2. 1b/mlx/config.json +62 -0
  3. 1b/mlx/llama_3_2_1b_mlx_int4.pte +3 -0
  4. 3b/mlx/config.json +62 -0
  5. 3b/mlx/llama_3_2_3b_mlx_int4.pte +3 -0
.gitattributes CHANGED
@@ -46,3 +46,5 @@ llama-3.2-3B/original/llama3_2_3B_bf16.pte filter=lfs diff=lfs merge=lfs -text
46
  1b/xnnpack/llama_3_2_1b_xnnpack_spinquant.pte filter=lfs diff=lfs merge=lfs -text
47
  3b/xnnpack/llama_3_2_3b_xnnpack_bf16.pte filter=lfs diff=lfs merge=lfs -text
48
  3b/xnnpack/llama_3_2_3b_xnnpack_spinquant.pte filter=lfs diff=lfs merge=lfs -text
 
 
 
46
  1b/xnnpack/llama_3_2_1b_xnnpack_spinquant.pte filter=lfs diff=lfs merge=lfs -text
47
  3b/xnnpack/llama_3_2_3b_xnnpack_bf16.pte filter=lfs diff=lfs merge=lfs -text
48
  3b/xnnpack/llama_3_2_3b_xnnpack_spinquant.pte filter=lfs diff=lfs merge=lfs -text
49
+ 1b/mlx/llama_3_2_1b_mlx_int4.pte filter=lfs diff=lfs merge=lfs -text
50
+ 3b/mlx/llama_3_2_3b_mlx_int4.pte filter=lfs diff=lfs merge=lfs -text
1b/mlx/config.json ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "$schema": "https://huggingface.co/software-mansion/react-native-executorch-spec/resolve/main/config.schema.json",
3
+ "model": "llama_3_2",
4
+ "family": "llama",
5
+ "capabilities": [
6
+ "text-generation"
7
+ ],
8
+ "backend": "mlx",
9
+ "license": "llama3.2",
10
+ "variants": [
11
+ {
12
+ "file": "llama_3_2_1b_mlx_int4.pte",
13
+ "precision": "4w",
14
+ "quantized": true,
15
+ "default": true,
16
+ "methods": {
17
+ "enable_dynamic_shape": true,
18
+ "forward": {
19
+ "inputs": [
20
+ {
21
+ "shape": [
22
+ 1,
23
+ {
24
+ "min": 1,
25
+ "max": 4095
26
+ }
27
+ ],
28
+ "dtype": "int64"
29
+ },
30
+ {
31
+ "shape": [
32
+ 1
33
+ ],
34
+ "dtype": "int64"
35
+ }
36
+ ],
37
+ "outputs": [
38
+ {
39
+ "shape": [
40
+ 1,
41
+ 128256
42
+ ],
43
+ "dtype": "bfloat16"
44
+ }
45
+ ]
46
+ },
47
+ "get_bos_id": 128000,
48
+ "get_eos_ids": [
49
+ 128009,
50
+ 128001
51
+ ],
52
+ "get_max_context_len": 4096,
53
+ "get_max_seq_len": 4096,
54
+ "get_n_layers": 16,
55
+ "get_vocab_size": 128256,
56
+ "use_kv_cache": true,
57
+ "use_sdpa_with_kv_cache": true
58
+ }
59
+ }
60
+ ],
61
+ "size": "1b"
62
+ }
1b/mlx/llama_3_2_1b_mlx_int4.pte ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8785362decb8c21484b5b91a09c21393509617d6dc0a61bc51ef82b659ef0c32
3
+ size 1182100864
3b/mlx/config.json ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "$schema": "https://huggingface.co/software-mansion/react-native-executorch-spec/resolve/main/config.schema.json",
3
+ "model": "llama_3_2",
4
+ "family": "llama",
5
+ "capabilities": [
6
+ "text-generation"
7
+ ],
8
+ "backend": "mlx",
9
+ "license": "llama3.2",
10
+ "variants": [
11
+ {
12
+ "file": "llama_3_2_3b_mlx_int4.pte",
13
+ "precision": "4w",
14
+ "quantized": true,
15
+ "default": true,
16
+ "methods": {
17
+ "enable_dynamic_shape": true,
18
+ "forward": {
19
+ "inputs": [
20
+ {
21
+ "shape": [
22
+ 1,
23
+ {
24
+ "min": 1,
25
+ "max": 4095
26
+ }
27
+ ],
28
+ "dtype": "int64"
29
+ },
30
+ {
31
+ "shape": [
32
+ 1
33
+ ],
34
+ "dtype": "int64"
35
+ }
36
+ ],
37
+ "outputs": [
38
+ {
39
+ "shape": [
40
+ 1,
41
+ 128256
42
+ ],
43
+ "dtype": "bfloat16"
44
+ }
45
+ ]
46
+ },
47
+ "get_bos_id": 128000,
48
+ "get_eos_ids": [
49
+ 128009,
50
+ 128001
51
+ ],
52
+ "get_max_context_len": 4096,
53
+ "get_max_seq_len": 4096,
54
+ "get_n_layers": 28,
55
+ "get_vocab_size": 128256,
56
+ "use_kv_cache": true,
57
+ "use_sdpa_with_kv_cache": true
58
+ }
59
+ }
60
+ ],
61
+ "size": "3b"
62
+ }
3b/mlx/llama_3_2_3b_mlx_int4.pte ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:25197cd6d8f9912f914b837c91a88656f24a6cc0cc2613c6e4f1074f10c96771
3
+ size 2495273728