How can I quantize DeepSeek-V3 with GPTQ quantization method using 8xH100 GPUs?

[ashgold]

How can I quantize DeepSeek-V3 with GPTQ quantization method using 8xH100 GPUs?
I tried but failed.

python codes

from pathlib import Path

import torch
from compressed_tensors.compressors import ModelCompressor
from compressed_tensors.quantization import (
    QuantizationConfig,
    QuantizationStatus,
    apply_quantization_config,
)
from datasets import load_dataset
from torch.utils.data import DataLoader, RandomSampler
from tqdm import tqdm
from transformers import AutoModelForCausalLM, AutoTokenizer, DefaultDataCollator

config_file = Path(__file__).parent / "gptq_int4_config.json"

MODEL_NAME = "DeepSeek-R1-bf16"
MODEL_ID = "/data/models/" + MODEL_NAME
SAVE_DIR = MODEL_ID + "-gptq-128g"

# Select calibration dataset.
DATASET_PATH = "mit-han-lab/pile-val-backup"
DATASET_FILE = "val.jsonl"

# Select number of samples. 512 samples is a good place to start.
# Increasing the number of samples can improve accuracy.
NUM_CALIBRATION_SAMPLES = 1
MAX_SEQUENCE_LENGTH = 2048
device = "auto"

# create dataset
# To solve the problem of always fetching N data from the beginning of the dataset, we fetch 100x data and sample it with a randomizer.
dataset = load_dataset(path=DATASET_PATH, data_files=DATASET_FILE, split=f"train[:{NUM_CALIBRATION_SAMPLES * 100}]")
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)

def preprocess(example):
    return {"text": tokenizer.apply_chat_template( [{"role": "user", "content": example["text"]}], tokenize=False)}
dataset = dataset.map(preprocess)

def tokenize_function(examples):
    return tokenizer(examples["text"], padding=False, truncation=True, max_length=MAX_SEQUENCE_LENGTH, add_special_tokens=False)
tokenized_dataset = dataset.map(tokenize_function, batched=True, remove_columns=dataset.column_names)

data_loader = DataLoader(
    tokenized_dataset, batch_size=1, collate_fn=DefaultDataCollator(), sampler=RandomSampler(tokenized_dataset)
)
print("Loading dataset finished.")


model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, 
    device_map=device, 
    torch_dtype="auto"
)
model.eval()  # no grad or updates needed for base model
print(model)
print("Loading model finished.")

config = QuantizationConfig.model_validate_json(config_file.read_text())

# set status to calibration
config.quantization_status = QuantizationStatus.CALIBRATION

# initialize quantization
apply_quantization_config(model, config)
print("Initializing quantization finished.")

with torch.no_grad():
    for idx, sample in tqdm(enumerate(data_loader), desc="Running calibration", total=NUM_CALIBRATION_SAMPLES):
        sample = {k: v.to(model.device) for k, v in sample.items()}
        _ = model(**sample)

        if idx >= NUM_CALIBRATION_SAMPLES:
            break
print("Calibration process finished.")

# convert model to QDQ model
compressor = ModelCompressor(quantization_config=config)
compressed_state_dict = compressor.compress(model, show_progress=True)
print("Compression process finished.")

# save QDQ model
model.save_pretrained(SAVE_DIR, state_dict=compressed_state_dict)
compressor.update_config(SAVE_DIR)
print("Saving compressed model finished.")

gptq_int4_config.json

{
	"quant_method": "compressed-tensors",
	"format": "pack-quantized",
    "ignore": [
      "lm_head",
      "re:.*self_attn.*",
      "re:.*shared_experts.*",
      "re:.*mlp\\.(gate|up|gate_up|down)_proj.*"
    ],
    "kv_cache_scheme": null,
	"global_compression_ratio": null,
	"config_groups": {
        "group_1": {
            "input_activations": null,
            "output_activations": null,
            "targets": ["Linear"],
            "weights": {
                "actorder": null,
                "block_structure": null,
                "dynamic": false,
                "group_size": 128,
                "num_bits": 4,
                "observer": "minmax",
                "observer_kwargs": {},
                "strategy": "group",
                "symmetric": true,
                "type": "int"
            }
        }
    }
}

below is the result.

Repo card metadata block was not found. Setting CardData to empty.
Loading dataset finished.
`rope_scaling`'s factor field must be a float >= 1, got 40
`rope_scaling`'s beta_fast field must be a float, got 32
`rope_scaling`'s beta_slow field must be a float, got 1
Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 135/135 [01:48<00:00,  1.24it/s]
Some parameters are on the meta device because they were offloaded to the cpu.
DeepseekV3ForCausalLM(
  (model): DeepseekV3Model(
    (embed_tokens): Embedding(129280, 7168)
    (layers): ModuleList(
      (0-2): 3 x DeepseekV3DecoderLayer(
        (self_attn): DeepseekV3Attention(
          (q_a_proj): Linear(in_features=7168, out_features=1536, bias=False)
          (q_a_layernorm): DeepseekV3RMSNorm((1536,), eps=1e-06)
          (q_b_proj): Linear(in_features=1536, out_features=24576, bias=False)
          (kv_a_proj_with_mqa): Linear(in_features=7168, out_features=576, bias=False)
          (kv_a_layernorm): DeepseekV3RMSNorm((512,), eps=1e-06)
          (kv_b_proj): Linear(in_features=512, out_features=32768, bias=False)
          (o_proj): Linear(in_features=16384, out_features=7168, bias=False)
        )
        (mlp): DeepseekV3MLP(
          (gate_proj): Linear(in_features=7168, out_features=18432, bias=False)
          (up_proj): Linear(in_features=7168, out_features=18432, bias=False)
          (down_proj): Linear(in_features=18432, out_features=7168, bias=False)
          (act_fn): SiLU()
        )
        (input_layernorm): DeepseekV3RMSNorm((7168,), eps=1e-06)
        (post_attention_layernorm): DeepseekV3RMSNorm((7168,), eps=1e-06)
      )
      (3-60): 58 x DeepseekV3DecoderLayer(
        (self_attn): DeepseekV3Attention(
          (q_a_proj): Linear(in_features=7168, out_features=1536, bias=False)
          (q_a_layernorm): DeepseekV3RMSNorm((1536,), eps=1e-06)
          (q_b_proj): Linear(in_features=1536, out_features=24576, bias=False)
          (kv_a_proj_with_mqa): Linear(in_features=7168, out_features=576, bias=False)
          (kv_a_layernorm): DeepseekV3RMSNorm((512,), eps=1e-06)
          (kv_b_proj): Linear(in_features=512, out_features=32768, bias=False)
          (o_proj): Linear(in_features=16384, out_features=7168, bias=False)
        )
        (mlp): DeepseekV3MoE(
          (experts): ModuleList(
            (0-255): 256 x DeepseekV3MLP(
              (gate_proj): Linear(in_features=7168, out_features=2048, bias=False)
              (up_proj): Linear(in_features=7168, out_features=2048, bias=False)
              (down_proj): Linear(in_features=2048, out_features=7168, bias=False)
              (act_fn): SiLU()
            )
          )
          (gate): DeepseekV3TopkRouter()
          (shared_experts): DeepseekV3MLP(
            (gate_proj): Linear(in_features=7168, out_features=2048, bias=False)
            (up_proj): Linear(in_features=7168, out_features=2048, bias=False)
            (down_proj): Linear(in_features=2048, out_features=7168, bias=False)
            (act_fn): SiLU()
          )
        )
        (input_layernorm): DeepseekV3RMSNorm((7168,), eps=1e-06)
        (post_attention_layernorm): DeepseekV3RMSNorm((7168,), eps=1e-06)
      )
    )
    (norm): DeepseekV3RMSNorm((7168,), eps=1e-06)
    (rotary_emb): DeepseekV3RotaryEmbedding()
  )
  (lm_head): Linear(in_features=7168, out_features=129280, bias=False)
)
Loading model finished.
Initializing quantization finished.
Running calibration: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:47<00:00, 47.94s/it]
Calibration process finished.








Traceback (most recent call last):
  File "/home/jovyan/workspace/awq/quantize_comressedtensors_gptq.py", line 116, in <module>
    compressed_state_dict = compressor.compress(model, show_progress=True)
                            ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jovyan/.local/lib/python3.12/site-packages/compressed_tensors/compressors/model_compressors/model_compressor.py", line 511, in compress
    state_dict = self.quantization_compressor.compress(
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jovyan/.local/lib/python3.12/site-packages/compressed_tensors/compressors/quantized_compressors/base.py", line 106, in compress
    compressed_dict[name] = value.to(save_device)
                            ^^^^^^^^^^^^^^^^^^^^^
NotImplementedError: Cannot copy out of meta tensor; no data!

[kylesayrs]

Hi @ashgold!

This is a work in progress for LLM Compressor. If you're interested in learning more, I recommend taking a look at vllm-project/llm-compressor#1263 and vllm-project/llm-compressor#1535 (comment)