大语言模型量化方法对比：GPTQ、GGUF、AWQ

 del model, tokenizer, pipe
 
 import torch
 torch.cuda.empty_cache()

 pip install git+https://github.com/huggingface/transformers.git
 pip install accelerate bitsandbytes xformers

 from torch import bfloat16
 from transformers import pipeline
 
 # Load in your LLM without any compression tricks
 pipe = pipeline(
     "text-generation", 
     model="HuggingFaceH4/zephyr-7b-beta", 
     torch_dtype=bfloat16, 
     device_map="auto"
 )

 messages = [
     {
         "role": "system",
         "content": "You are a friendly chatbot.",
     },
     {
         "role": "user", 
         "content": "Tell me a funny joke about Large Language Models."
     },
 ]
 prompt = pipe.tokenizer.apply_chat_template(
     messages, 
     tokenize=False, 
     add_generation_prompt=True
 )

 outputs = pipe(
     prompt, 
     max_new_tokens=256, 
     do_sample=True, 
     temperature=0.1, 
     top_p=0.95
 )
 print(outputs[0]["generated_text"])

 from accelerate import Accelerator
 
 # Shard our model into pieces of 1GB
 accelerator = Accelerator()
 accelerator.save_model(
     model=pipe.model, 
     save_directory="/content/model", 
     max_shard_size="4GB"
 )

 from transformers import BitsAndBytesConfig
 from torch import bfloat16
 
 # Our 4-bit configuration to load the LLM with less GPU memory
 bnb_config = BitsAndBytesConfig(
     load_in_4bit=True,  # 4-bit quantization
     bnb_4bit_quant_type='nf4',  # Normalized float 4
     bnb_4bit_use_double_quant=True,  # Second quantization after the first
     bnb_4bit_compute_dtype=bfloat16  # Computation type
 )

 from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
 
 # Zephyr with BitsAndBytes Configuration
 tokenizer = AutoTokenizer.from_pretrained("HuggingFaceH4/zephyr-7b-alpha")
 model = AutoModelForCausalLM.from_pretrained(
     "HuggingFaceH4/zephyr-7b-alpha",
     quantization_config=bnb_config,
     device_map='auto',
 )
 
 # Create a pipeline
 pipe = pipeline(model=model, tokenizer=tokenizer, task='text-generation')

 outputs = pipe(
     prompt, 
     max_new_tokens=256, 
     do_sample=True, 
     temperature=0.7, 
     top_p=0.95
 )
 print(outputs[0]["generated_text"])

 pip install optimum
 pip install auto-gptq --extra-index-url https://huggingface.github.io/autogptq-index/whl/cu118/

 from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
 
 # Load LLM and Tokenizer
 model_id = "TheBloke/zephyr-7B-beta-GPTQ"
 tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True)
 model = AutoModelForCausalLM.from_pretrained(
     model_id,
     device_map="auto",
     trust_remote_code=False,
     revision="main"
 )
 
 # Create a pipeline
 pipe = pipeline(model=model, tokenizer=tokenizer, task='text-generation')

 pip install ctransformers[cuda]

 from ctransformers import AutoModelForCausalLM
 from transformers import AutoTokenizer, pipeline
 
 # Load LLM and Tokenizer
 # Use `gpu_layers` to specify how many layers will be offloaded to the GPU.
 model = AutoModelForCausalLM.from_pretrained(
     "TheBloke/zephyr-7B-beta-GGUF",
     model_file="zephyr-7b-beta.Q4_K_M.gguf",
     model_type="mistral", gpu_layers=50, hf=True
 )
 tokenizer = AutoTokenizer.from_pretrained(
     "HuggingFaceH4/zephyr-7b-beta", use_fast=True
 )
 
 # Create a pipeline
 pipe = pipeline(model=model, tokenizer=tokenizer, task='text-generation')

 outputs = pipe(prompt, max_new_tokens=256)
 print(outputs[0]["generated_text"])

 pip install vllm

 from vllm import LLM, SamplingParams
 
 # Load the LLM
 sampling_params = SamplingParams(temperature=0.0, top_p=1.0, max_tokens=256)
 llm = LLM(
     model="TheBloke/zephyr-7B-beta-AWQ", 
     quantization='awq', 
     dtype='half', 
     gpu_memory_utilization=.95, 
     max_model_len=4096
 )

 output = llm.generate(prompt, sampling_params)
 print(output[0].outputs[0].text)