Python底层技术揭秘：如何实现情感分析

import nltk
from nltk.corpus import movie_reviews

nltk.download('movie_reviews')

from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer

nltk.download('stopwords')
nltk.download('punkt')
nltk.download('wordnet')

def preprocess_text(text):
    stop_words = set(stopwords.words('english'))
    lemmatizer = WordNetLemmatizer()
    
    tokens = word_tokenize(text.lower())
    tokens = [lemmatizer.lemmatize(token) for token in tokens if token.isalpha()]
    tokens = [token for token in tokens if token not in stop_words]
    
    return tokens

from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

def train_model(data, labels):
    vectorizer = CountVectorizer(tokenizer=preprocess_text)
    features = vectorizer.fit_transfORM(data)
    
    X_train, X_test, y_train, y_test = train_test_split(features, labels, test_size=0.2, random_state=42)
    
    model = MultinomialNB()
    model.fit(X_train, y_train)
    
    return model, vectorizer, X_test, y_test

def predict_sentiment(model, vectorizer, text):
    tokens = preprocess_text(text)
    features = vectorizer.transform([' '.join(tokens)])
    sentiment = model.predict(features)
    
    return sentiment[0]

# 使用电影评论数据集进行情感分析的训练和预测
data = [movie_reviews.raw(fileid) for fileid in movie_reviews.fileids()]
labels = [movie_reviews.cateGories(fileid)[0] for fileid in movie_reviews.fileids()]

model, vectorizer, X_test, y_test = train_model(data, labels)
y_pred = model.predict(X_test)

print('Accuracy:', accuracy_score(y_test, y_pred))