多模态Agent开发：Python打造超酷智能交互系统

from PIL import Image

image = Image.open("example.jpg")

image.show()

import speech_recognition as sr

r = sr.Recognizer()

with sr.Microphone() as source:

   print("请说话...")

   audio = r.listen(source)

try:

   text = r.recognize_google(audio)

   print("识别结果：", text)

except sr.UnknownValueError:

   print("无法识别语音")

except sr.RequestError as e:

   print("请求错误：", e)

import nltk

from nltk.tokenize import word_tokenize

from nltk.probability import FreqDist

from nltk.corpus import stopwords

from nltk.stem import WordNetLemmatizer

from nltk.tag import pos_tag

text = "I love programming in Python"

tokens = word_tokenize(text)

tagged = pos_tag(tokens)

print(tagged)

pip install torch torchvision transformers

import torch

from torchvision import transforms

from transformers import VisionEncoderDecoderModel, AutoTokenizer

# 加载预训练模型和分词器

model = VisionEncoderDecoderModel.from_pretrained("nlpconnect/vit-gpt2-image-captioning")

tokenizer = AutoTokenizer.from_pretrained("nlpconnect/vit-gpt2-image-captioning")

# 图像预处理

transform = transforms.Compose([

   transforms.Resize((224, 224)),

   transforms.ToTensor(),

   transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])

])

def generate_caption(image_path):

   # 加载图像

   image = Image.open(image_path)

   image = transform(image).unsqueeze(0)

   # 生成描述

   output = model.generate(image)

   caption = tokenizer.decode(output[0], skip_special_tokens=True)

   return caption

# 示例用法

image_path = "example.jpg"

caption = generate_caption(image_path)

print("图像描述：", caption)

import speech_recognition as sr

import os

def recognize_and_execute():

   r = sr.Recognizer()

   with sr.Microphone() as source:

       print("请说出指令...")

       audio = r.listen(source)

   try:

       text = r.recognize_google(audio)

       print("识别结果：", text)

       if "打开计算器" in text.lower():

           os.system("start calc.exe")  # 在Windows系统下打开计算器

   except sr.UnknownValueError:

       print("无法识别语音")

   except sr.RequestError as e:

       print("请求错误：", e)

# 调用函数

recognize_and_execute()

pip install spacy

python -m spacy download en_core_web_sm

import spacy

nlp = spacy.load("en_core_web_sm")

def analyze_sentiment_and_reply(text):

   doc = nlp(text)

   sentiment = 0

   for token in doc:

       if token.has_vector and token.is_alpha:

           sentiment += token.sentiment

   if sentiment > 0:

       reply = "听起来你心情不错呀！"

   elif sentiment < 0:

       reply = "别不开心啦，一切都会好起来的！"

   else:

       reply = "感觉你心情很平静呢。"

   return reply

# 示例用法

input_text = "I had a great day today!"

reply = analyze_sentiment_and_reply(input_text)

print("回复：", reply)

pip install torch torchvision transformers

import torch

import torch.nn as nn

from torchvision import datasets, transforms

from transformers import AutoTokenizer, AutoModel

from torch.utils.data import DataLoader, TensorDataset

# 图像预处理

image_transform = transforms.Compose([

   transforms.Resize((224, 224)),

   transforms.ToTensor(),

   transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])

])

# 加载图像数据集

image_dataset = datasets.ImageFolder(root='image_data', transform=image_transform)

image_loader = DataLoader(image_dataset, batch_size=32, shuffle=True)

# 加载文本数据集

tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')

text_data = []

labels = []

with open('text_data.txt', 'r', encoding='utf-8') as f:

   for line in f:

       parts = line.strip().split('t')

       text_data.append(parts[0])

       labels.append(int(parts[1]))

encoded_text = tokenizer(text_data, padding=True, truncation=True, return_tensors='pt')

text_tensor = encoded_text['input_ids']

label_tensor = torch.tensor(labels)

text_dataset = TensorDataset(text_tensor, label_tensor)

text_loader = DataLoader(text_dataset, batch_size=32, shuffle=True)

# 定义图像特征提取器

class ImageFeatureExtractor(nn.Module):

   def __init__(self):

       super(ImageFeatureExtractor, self).__init__()

       self.model = torchvision.models.resnet18(pretrained=True)

       self.model.fc = nn.Identity()

   def forward(self, x):

       return self.model(x)

# 定义文本特征提取器

class TextFeatureExtractor(nn.Module):

   def __init__(self):

       super(TextFeatureExtractor, self).__init__()

       self.model = AutoModel.from_pretrained('bert-base-uncased')

   def forward(self, x):

       outputs = self.model(x)

       return outputs.pooler_output

# 定义融合模型

class FusionModel(nn.Module):

   def __init__(self, image_feature_size, text_feature_size, num_classes):

       super(FusionModel, self).__init__()

       self.image_feature_extractor = ImageFeatureExtractor()

       self.text_feature_extractor = TextFeatureExtractor()

       self.fc = nn.Sequential(

           nn.Linear(image_feature_size + text_feature_size, 128),

           nn.ReLU(),

           nn.Linear(128, num_classes)

       )

   def forward(self, image, text):

       image_features = self.image_feature_extractor(image)

       text_features = self.text_feature_extractor(text)

       fused_features = torch.cat((image_features, text_features), dim=1)

       return self.fc(fused_features)

# 训练模型

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

model = FusionModel(image_feature_size=512, text_feature_size=768, num_classes=2).to(device)

criterion = nn.CrossEntropyLoss()

optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

for epoch in range(10):

   for i, (images, (texts, labels)) in enumerate(zip(image_loader, text_loader)):

       images = images.to(device)

       texts = texts.to(device)

       labels = labels.to(device)

       optimizer.zero_grad()

       outputs = model(images, texts)

       loss = criterion(outputs, labels)

       loss.backward()

       optimizer.step()

       if (i + 1) % 10 == 0:

           print(f'Epoch [{epoch + 1}/10], Step [{i + 1}/{len(image_loader)}], Loss: {loss.item():.4f}')