From c348e2ca84817135bce25f4e49ce5777702bc40e Mon Sep 17 00:00:00 2001 From: DaoYu Lin <498435408@qq.com> Date: Wed, 9 May 2018 04:36:41 +0000 Subject: [PATCH 1/2] Update requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 0a494a8..bde8bcc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ h5py==2.6.0 numpy==1.11.2 pandas==0.18.1 Theano==0.8.2 +tensorflow-gpu==1.1.0 From f0b4fdb0bbf2fb7e2fa03e323aff6ce74a66ad9b Mon Sep 17 00:00:00 2001 From: DY <498435408@qq.com> Date: Tue, 11 Aug 2026 16:29:19 +0800 Subject: [PATCH 2/2] Add modern Python 3 sentiment baseline (#24) --- .gitignore | 6 + CONTRIBUTING.md | 24 ++++ README.md | 103 +++++++++++------ SECURITY.md | 15 +++ requirements-legacy.txt | 11 ++ requirements.txt | 13 +-- sentiment_cli.py | 224 ++++++++++++++++++++++++++++++++++++ tests/test_sentiment_cli.py | 60 ++++++++++ 8 files changed, 410 insertions(+), 46 deletions(-) create mode 100644 .gitignore create mode 100644 CONTRIBUTING.md create mode 100644 SECURITY.md create mode 100644 requirements-legacy.txt create mode 100644 sentiment_cli.py create mode 100644 tests/test_sentiment_cli.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..934a9ab --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.DS_Store +.venv/ +__pycache__/ +*.py[cod] +artifacts/ +.pytest_cache/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..5f35082 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,24 @@ +# Contributing + +Contributions that improve reproducibility, evaluation, documentation, and modern Python compatibility are welcome. + +## Development setup + +```bash +python3 -m venv .venv +source .venv/bin/activate +python -m pip install -r requirements.txt +python -m unittest discover -s tests -v +``` + +## Pull requests + +- Keep legacy Python 2 files available for historical reference unless a migration includes a documented replacement. +- Add tests for command-line or model-format changes. +- Do not commit locally trained models, virtual environments, credentials, or private datasets. +- Report evaluation results with the random seed, split strategy, data version, and metric definition. +- Document any new data source, its license, and its privacy constraints. + +## Model changes + +A higher score on the bundled data is not sufficient on its own. Explain possible leakage, class balance, domain assumptions, and how the model behaves on unseen data. diff --git a/README.md b/README.md index 75ec6e3..c323e7c 100644 --- a/README.md +++ b/README.md @@ -1,51 +1,80 @@ -#环境要求 -- Unix/Linux系统 -- python 2.7 -- python包安装： keras,sklearn,gensim,jieba,h5py,numpy,pandas -``` -sudo pip install -r requirements.txt -``` -# 用法 +# Chinese Sentiment Analysis -## 使用SVM分类器进行情感分类： -``` -python predict.py svm 这个手机质量很好，我很喜欢，不错不错 +中文商品评论情感分类示例。仓库保留了 2016 年的 Word2Vec + SVM/LSTM 实现和预训练文件，并新增了一个面向现代 Python 3 环境的、可重新训练的 TF-IDF + Linear SVM 基线。 -``` -``` -python predict.py svm 这书的印刷质量的太差了吧，还有缺页，以后再也不买了 +## 项目状态 -``` +原始代码依赖 Python 2.7、Keras 1、TensorFlow 1 和早期版本的 gensim/scikit-learn。由于这些框架及序列化格式已经发生重大变化，旧模型仅作为研究和复现实验的历史资料保留，不应直接加载不可信的 `.pkl`、`.h5` 或 YAML 文件。 -## 使用LSTM进行情感分类： +新的 [`sentiment_cli.py`](sentiment_cli.py) 具备以下特点： + +- 支持 Python 3.9 及以上版本； +- 使用字符级 TF-IDF，不依赖中文分词词典； +- 使用 Linear SVM 训练和预测； +- 从仓库现有正负评论数据重新训练模型； +- 保存训练时间、数据量、评估准确率和运行环境等元数据； +- 默认不提交生成的模型文件，避免把本地序列化产物误当成可信发布物。 + +## 快速开始 + +创建独立环境并安装现代依赖： + +```bash +python3 -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip +python -m pip install -r requirements.txt ``` -python predict.py lstm 酒店的环境不错，空间挺大的 + +训练模型： + +```bash +python sentiment_cli.py train ``` + +默认模型会写入 `artifacts/sentiment_svm.joblib`。训练命令会在固定随机种子下划分测试集，并打印 JSON 格式的评估摘要。 + +预测一条或多条评论： + +```bash +python sentiment_cli.py predict "这个手机质量很好，我很喜欢" +python sentiment_cli.py predict "酒店环境不错" "电脑散热太差了" --json ``` -python predict.py lstm 电脑散热太差，偶尔还蓝屏，送货也太慢了吧 + +指定模型路径： + +```bash +python sentiment_cli.py train --model /path/to/model.joblib +python sentiment_cli.py predict --model /path/to/model.joblib "物流很快" ``` -#程序 -- code/Sentiment_lstm.py 使用word2vec和LSTM训练和预测 -- code/Sentiment_svm.py 使用word2vec和svm训练和预测 -- predict.py 调用Sentiment_lstm.py及Sentiment_svm.py进行预测 +## 数据 + +- [`data/pos.xls`](data/pos.xls)：正向评论。 +- [`data/neg.xls`](data/neg.xls)：负向评论。 -#数据 -- ./data/ 原始数据文件夹 - - data/neg.xls 负样本原始数据 - - data/pos.xls 正样本原始数据 +这些数据来自早期实验，仅适合教学和基线比较。实际应用前应重新确认来源、许可、领域偏差和隐私要求，并使用目标领域数据重新评估。 + +## 旧版实现 + +- [`code/Sentiment_svm.py`](code/Sentiment_svm.py)：Word2Vec + RBF SVM。 +- [`code/Sentiment_lstm.py`](code/Sentiment_lstm.py)：Word2Vec + Keras LSTM。 +- [`predict.py`](predict.py)：旧版 Python 2 预测入口。 +- [`requirements-legacy.txt`](requirements-legacy.txt)：原始依赖版本记录，仅用于历史研究。 + +旧版模型与现代库通常不兼容。推荐通过新的 Python 3 CLI 重新训练，而不是尝试在生产环境中恢复旧运行栈。 + +## 测试 + +```bash +python -m unittest discover -s tests -v +python -m compileall sentiment_cli.py tests +``` -- ./svm_data/ svm数据文件夹 - - ./svm_data/\*.npy 处理后的训练数据和测试数据 - - ./svm_data/svm_model/ 保存训练好的svm模型 - - ./svm_data/w2v_model/ 保存训练好的word2vec模型 +## 参与维护 +欢迎改进评估方法、数据说明、命令行体验和现代模型基线。提交前请阅读 [CONTRIBUTING.md](CONTRIBUTING.md)。处理模型文件或数据时请同时阅读 [SECURITY.md](SECURITY.md)。 -- ./lstm_data/ lstm数据文件夹 - - ./lstm_data/Word2vec_model.pkl 保存训练好的word2vec模型 - - ./lstm_data/lstm.yml 保存训练网络的结构 - - ./lstm_data/lstm.h5 保存网络训练到的权重 - -#详细介绍 +## 背景文章 -[购物评论情感分析的实现](http://buptldy.github.io/2016/07/20/2016-07-20-sentiment%20analysis/) +[购物评论情感分析的实现](https://buptldy.github.io/2016/07/20/2016-07-20-sentiment%20analysis/) diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..6a051c8 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,15 @@ +# Security Policy + +## Serialized models + +Python pickle and joblib files can execute code while loading. Keras HDF5/YAML artifacts may also depend on unsafe or obsolete deserialization behavior. + +- Only load model files you created locally or obtained from a trusted source. +- Do not accept arbitrary model uploads in a service that calls this repository's loaders. +- Retrain the modern baseline from the documented dataset when provenance is uncertain. + +## Data and model limitations + +The bundled review data is historical and may contain biased, offensive, or privacy-sensitive text. Do not use the example model for consequential decisions without a separate data review, representative evaluation, and appropriate human oversight. + +Please report vulnerabilities privately to the repository owner before publishing exploit details. diff --git a/requirements-legacy.txt b/requirements-legacy.txt new file mode 100644 index 0000000..513d624 --- /dev/null +++ b/requirements-legacy.txt @@ -0,0 +1,11 @@ +# Historical environment from the original Python 2 implementation. +# These versions are retained for reference and are not recommended for new setups. +Keras==1.1.1 +gensim==0.13.3 +jieba==0.38 +sklearn==0.0 +h5py==2.6.0 +numpy==1.11.2 +pandas==0.18.1 +Theano==0.8.2 +tensorflow-gpu==1.1.0 diff --git a/requirements.txt b/requirements.txt index bde8bcc..1e53fe1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,4 @@ -Keras==1.1.1 -gensim==0.13.3 -jieba==0.38 -sklearn==0.0 -h5py==2.6.0 -numpy==1.11.2 -pandas==0.18.1 -Theano==0.8.2 -tensorflow-gpu==1.1.0 +joblib>=1.3,<2 +pandas>=2.0,<3 +scikit-learn>=1.3,<2 +xlrd>=2.0,<3 diff --git a/sentiment_cli.py b/sentiment_cli.py new file mode 100644 index 0000000..6f21567 --- /dev/null +++ b/sentiment_cli.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +"""Train and run a modern Chinese sentiment-classification baseline.""" + +from __future__ import annotations + +import argparse +import json +import platform +from datetime import datetime, timezone +from pathlib import Path +from typing import Iterable + + +MODEL_FORMAT_VERSION = 1 +DEFAULT_DATA_DIRECTORY = Path(__file__).resolve().parent / "data" +DEFAULT_MODEL_PATH = Path(__file__).resolve().parent / "artifacts" / "sentiment_svm.joblib" + + +def clean_reviews(values: Iterable[object]) -> list[str]: + """Convert spreadsheet values to non-empty, normalized strings.""" + + reviews: list[str] = [] + for value in values: + if value is None: + continue + text = str(value).replace("\u3000", " ").strip() + if text and text.lower() != "nan": + reviews.append(text) + return reviews + + +def load_dataset(data_directory: Path) -> tuple[list[str], list[int]]: + """Load positive and negative reviews from the repository XLS files.""" + + import pandas as pd + + positive_path = data_directory / "pos.xls" + negative_path = data_directory / "neg.xls" + missing = [str(path) for path in (positive_path, negative_path) if not path.is_file()] + if missing: + raise FileNotFoundError(f"Missing dataset file(s): {', '.join(missing)}") + + positive = clean_reviews( + pd.read_excel(positive_path, header=None, engine="xlrd").iloc[:, 0] + ) + negative = clean_reviews( + pd.read_excel(negative_path, header=None, engine="xlrd").iloc[:, 0] + ) + if not positive or not negative: + raise ValueError("Both positive and negative datasets must contain reviews.") + + reviews = positive + negative + labels = [1] * len(positive) + [0] * len(negative) + return reviews, labels + + +def build_pipeline(*, min_df: int = 2, max_features: int = 150_000): + """Build a deterministic character n-gram sentiment pipeline.""" + + from sklearn.feature_extraction.text import TfidfVectorizer + from sklearn.pipeline import Pipeline + from sklearn.svm import LinearSVC + + return Pipeline( + [ + ( + "tfidf", + TfidfVectorizer( + analyzer="char", + ngram_range=(2, 4), + min_df=min_df, + max_features=max_features, + sublinear_tf=True, + ), + ), + ("classifier", LinearSVC(dual="auto", random_state=42)), + ] + ) + + +def train_model( + *, + data_directory: Path, + model_path: Path, + test_size: float, + random_state: int, +) -> dict[str, object]: + """Train, evaluate, and persist a model plus reproducibility metadata.""" + + import joblib + import sklearn + from sklearn.metrics import accuracy_score + from sklearn.model_selection import train_test_split + + reviews, labels = load_dataset(data_directory) + train_reviews, test_reviews, train_labels, test_labels = train_test_split( + reviews, + labels, + test_size=test_size, + random_state=random_state, + stratify=labels, + ) + + pipeline = build_pipeline() + pipeline.fit(train_reviews, train_labels) + predicted_labels = pipeline.predict(test_reviews) + accuracy = float(accuracy_score(test_labels, predicted_labels)) + + metadata: dict[str, object] = { + "trained_at": datetime.now(timezone.utc).isoformat(), + "python_version": platform.python_version(), + "scikit_learn_version": sklearn.__version__, + "total_reviews": len(reviews), + "training_reviews": len(train_reviews), + "test_reviews": len(test_reviews), + "test_accuracy": round(accuracy, 6), + "test_size": test_size, + "random_state": random_state, + "labels": {"0": "negative", "1": "positive"}, + "features": "character TF-IDF n-grams (2-4)", + "classifier": "LinearSVC", + } + payload = { + "format_version": MODEL_FORMAT_VERSION, + "pipeline": pipeline, + "metadata": metadata, + } + + model_path.parent.mkdir(parents=True, exist_ok=True) + joblib.dump(payload, model_path) + return {"model_path": str(model_path.resolve()), **metadata} + + +def load_model(model_path: Path) -> dict[str, object]: + """Load a model produced by this CLI and validate its format.""" + + import joblib + + if not model_path.is_file(): + raise FileNotFoundError( + f"Model not found: {model_path}. Run `python sentiment_cli.py train` first." + ) + + payload = joblib.load(model_path) + if not isinstance(payload, dict) or payload.get("format_version") != MODEL_FORMAT_VERSION: + raise ValueError("Unsupported model format. Retrain with the current CLI.") + if "pipeline" not in payload or "metadata" not in payload: + raise ValueError("Model payload is incomplete. Retrain with the current CLI.") + return payload + + +def predict(model_path: Path, texts: list[str]) -> list[dict[str, object]]: + """Predict labels and decision scores for input texts.""" + + payload = load_model(model_path) + pipeline = payload["pipeline"] + labels = pipeline.predict(texts) + scores = pipeline.decision_function(texts) + return [ + { + "text": text, + "label": "positive" if int(label) == 1 else "negative", + "score": round(float(score), 6), + } + for text, label, score in zip(texts, labels, scores) + ] + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Train and run a Chinese sentiment-classification baseline." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + train_parser = subparsers.add_parser("train", help="Train and evaluate a model.") + train_parser.add_argument( + "--data-dir", type=Path, default=DEFAULT_DATA_DIRECTORY, help="XLS data directory." + ) + train_parser.add_argument( + "--model", type=Path, default=DEFAULT_MODEL_PATH, help="Output model path." + ) + train_parser.add_argument("--test-size", type=float, default=0.2) + train_parser.add_argument("--random-state", type=int, default=42) + + predict_parser = subparsers.add_parser("predict", help="Predict one or more reviews.") + predict_parser.add_argument("texts", nargs="+", help="Review text to classify.") + predict_parser.add_argument( + "--model", type=Path, default=DEFAULT_MODEL_PATH, help="Trained model path." + ) + predict_parser.add_argument( + "--json", action="store_true", help="Print machine-readable JSON output." + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + if args.command == "train": + if not 0 < args.test_size < 1: + raise ValueError("--test-size must be between 0 and 1.") + result = train_model( + data_directory=args.data_dir, + model_path=args.model, + test_size=args.test_size, + random_state=args.random_state, + ) + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 + + results = predict(args.model, args.texts) + if args.json: + print(json.dumps(results, ensure_ascii=False, indent=2)) + else: + for result in results: + print(f"{result['label']}\t{result['score']:+.6f}\t{result['text']}") + return 0 + except (FileNotFoundError, ValueError) as error: + build_parser().error(str(error)) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_sentiment_cli.py b/tests/test_sentiment_cli.py new file mode 100644 index 0000000..41a51b4 --- /dev/null +++ b/tests/test_sentiment_cli.py @@ -0,0 +1,60 @@ +import importlib.util +import io +import tempfile +import unittest +from contextlib import redirect_stderr +from pathlib import Path + +import sentiment_cli + + +HAS_SKLEARN = importlib.util.find_spec("sklearn") is not None + + +class CleanReviewsTest(unittest.TestCase): + def test_clean_reviews_removes_empty_values_and_normalizes_spaces(self): + self.assertEqual( + sentiment_cli.clean_reviews([None, " 很好 ", "\u3000一般\u3000", "", "nan"]), + ["很好", "一般"], + ) + + def test_parser_requires_a_subcommand(self): + with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + sentiment_cli.build_parser().parse_args([]) + + +@unittest.skipUnless(HAS_SKLEARN, "scikit-learn is not installed") +class PipelineTest(unittest.TestCase): + def test_pipeline_can_fit_and_predict_small_chinese_dataset(self): + positive = ["质量很好", "非常满意", "物流很快", "值得推荐"] + negative = ["质量太差", "非常失望", "物流很慢", "不会再买"] + pipeline = sentiment_cli.build_pipeline(min_df=1, max_features=1_000) + pipeline.fit(positive + negative, [1] * len(positive) + [0] * len(negative)) + + predictions = pipeline.predict(["质量很好", "质量太差"]) + + self.assertEqual(predictions.tolist(), [1, 0]) + + def test_model_round_trip_validates_format(self): + import joblib + + pipeline = sentiment_cli.build_pipeline(min_df=1, max_features=1_000) + pipeline.fit(["非常好", "真的喜欢", "非常差", "真的讨厌"], [1, 1, 0, 0]) + + with tempfile.TemporaryDirectory() as temporary_directory: + model_path = Path(temporary_directory) / "model.joblib" + joblib.dump( + { + "format_version": sentiment_cli.MODEL_FORMAT_VERSION, + "pipeline": pipeline, + "metadata": {"test": True}, + }, + model_path, + ) + predictions = sentiment_cli.predict(model_path, ["非常好", "非常差"]) + + self.assertEqual([item["label"] for item in predictions], ["positive", "negative"]) + + +if __name__ == "__main__": + unittest.main()