NLP文本分类全流程指南：从数据清理到模型落地的实用方法

import re
def clean_text(text):
    # 去掉特殊符号和数字
    text = re.sub(r'[^u4e00-u9fa5a-zA-Z]', ' ', text)
    # 去掉多余空格
    text = re.sub(r's+', ' ', text).strip()
    return text
# 示例：把"我买了10斤苹果！[广告]好吃到哭😭"变成"我买了斤苹果 好吃到哭"

import jieba
jieba.add_word("Python")  # 自定义词典，避免英文被拆分
def cut_chinese(text):
    return [word for word in jieba.cut(text) if len(word) > 1]  # 过滤单字
# 示例："我用Python做NLP文本分类" → ["用", "Python", "做", "NLP", "文本分类"]

stopwords = set(open("stopwords.txt", "r", encoding="utf-8").read().splitlines())
def remove_stopwords(words):
    return [word for word in words if word not in stopwords]

from sklearn.feature_extraction.text import TfidfVectorizer
# 示例文本
texts = ["我喜欢苹果", "我讨厌香蕉", "苹果好吃", "香蕉难吃"]
# 初始化TF-IDF
tfidf = TfidfVectorizer()
# 生成特征矩阵
features = tfidf.fit_transform(texts)
# 查看特征词和权重
print(tfidf.get_feature_names_out())  # ["苹果", "喜欢", "讨厌", "好吃", "香蕉", "难吃"]
print(features.toarray())  # 每行是一个文本的TF-IDF向量

from gensim.models import KeyedVectors
# 加载预训练模型（需要先下载：https://code.google.com/archive/p/word2vec/）
model = KeyedVectors.load_word2vec_format("GoogleNews-vectors-negative300.bin", binary=True)
# 查看“apple”的向量
print(model["apple"].shape)  # (300,)
# 找相似词
print(model.most_similar("apple", topn=3))  # [("banana", 0.72), ("orange", 0.68), ("pear", 0.65)]

from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import accuracy_score
# 训练数据（正面：1，负面：0）
X_train = ["我喜欢这个产品", "这个产品太烂了", "质量很好", "服务很差"]
y_train = [1, 0, 1, 0]
# 特征工程（TF-IDF）
tfidf = TfidfVectorizer()
X_train_vec = tfidf.fit_transform(X_train)
# 训练模型
nb_model = MultinomialNB()
nb_model.fit(X_train_vec, y_train)
# 测试数据
X_test = ["这个产品不错", "服务太糟糕了"]
X_test_vec = tfidf.transform(X_test)
# 预测
y_pred = nb_model.predict(X_test_vec)
print(accuracy_score([1,0], y_pred))  # 输出1.0（全对）

from keras.models import Sequential
from keras.layers import Embedding, Conv1D, GlobalMaxPooling1D, Dense
from keras.preprocessing.sequence import pad_sequences
from keras.preprocessing.text import Tokenizer
# 1. 准备数据
texts = ["我喜欢这个电影", "这个电影太无聊了", "剧情很精彩", "演员演技差"]
labels = [1, 0, 1, 0]
# 2.  tokenizer（把词变成索引）
tokenizer = Tokenizer(num_words=1000)
tokenizer.fit_on_texts(texts)
X = tokenizer.texts_to_sequences(texts)
# 3. 填充序列（让所有文本长度一致）
X = pad_sequences(X, maxlen=10)
# 4. 搭CNN模型
model = Sequential()
model.add(Embedding(input_dim=1000, output_dim=128, input_length=10))  # 词嵌入层
model.add(Conv1D(filters=64, kernel_size=3, activation='relu'))  # 卷积层（捕捉3-gram特征）
model.add(GlobalMaxPooling1D())  # 全局池化层（把卷积结果压缩成一个向量）
model.add(Dense(1, activation='sigmoid'))  # 输出层（二分类）
# 5. 编译模型
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
# 6. 训练
model.fit(X, labels, epochs=10, batch_size=2)

from transformers import BertTokenizer, TFBertForSequenceClassification
import tensorflow as tf
# 1. 加载预训练模型和tokenizer
tokenizer = BertTokenizer.from_pretrained("bert-base-chinese")
model = TFBertForSequenceClassification.from_pretrained("bert-base-chinese", num_labels=2)
# 2. 准备数据
texts = ["我喜欢这个产品", "这个产品太烂了"]
labels = [1, 0]
# 3.  tokenize（把文本变成BERT需要的输入格式：input_ids、attention_mask）
inputs = tokenizer(texts, padding=True, truncation=True, max_length=512, return_tensors="tf")
# 4. 编译模型
model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=2e-5), loss=model.hf_compute_loss, metrics=['accuracy'])
# 5. 训练
model.fit(dict(inputs), labels, epochs=3, batch_size=2)

from fastapi import FastAPI
import pickle
import jieba
# 1. 加载模型和tokenizer
with open("text_classifier.pkl", "rb") as f:
    model, vectorizer = pickle.load(f)
# 2. 初始化FastAPI
app = FastAPI()
# 3. 定义预处理函数
def preprocess(text):
    text = clean_text(text)  # 之前写的清洗函数
    words = cut_chinese(text)  # 之前写的分词函数
    words = remove_stopwords(words)  # 之前写的去停用词函数
    return " ".join(words)
# 4. 定义API接口
@app.post("/classify")
def classify(text: str):
    # 预处理
    processed_text = preprocess(text)
    # 特征转换
    features = vectorizer.transform([processed_text])
    # 预测
    prediction = model.predict(features)[0]
    # 返回结果
    return {
        "text": text,
        "processed_text": processed_text,
        "prediction": "正面" if prediction == 1 else "负面"
    }
# 运行：uvicorn main:app --reload（在终端输入）