From 24d16d7829403aae2a7ca1df424be0a64c6ba1a6 Mon Sep 17 00:00:00 2001 From: dingdingcar <1054263492@qq.com> Date: Sat, 7 Nov 2026 11:19:40 -0800 Subject: [PATCH] fix: script output survives a console encoding such as cp1252 Scripts printed non-ASCII JSON to stdout, which raises UnicodeEncodeError when an agent reads them through a pipe on Windows without UTF-8 mode. JSON on stdout is now ASCII-escaped in deeppapernote, paper-glossary, and research-memory; files written with --output stay UTF-8. run_pipeline's name=path lines switch stdout to UTF-8 instead, since escaped paths would not be usable. Tests run the scripts under PYTHONIOENCODING=cp1252. Co-Authored-By: Claude Opus 5.5 --- skills/deeppapernote/scripts/common.py | 6 +- skills/deeppapernote/scripts/run_pipeline.py | 9 ++- skills/deeppapernote/scripts/run_state.py | 2 +- .../scripts/user_configuration.py | 2 +- .../scripts/configure_glossary.py | 6 +- .../paper-glossary/scripts/glossary_common.py | 6 +- .../scripts/aggregate_reading.py | 6 +- .../scripts/write_research_memory.py | 4 +- .../tests/test_research_memory_aggregate.py | 10 ++++ .../tests/test_research_memory_write.py | 3 + tests/test_console_encoding.py | 56 +++++++++++++++++++ 11 files changed, 93 insertions(+), 17 deletions(-) create mode 100644 tests/test_console_encoding.py diff --git a/skills/deeppapernote/scripts/common.py b/skills/deeppapernote/scripts/common.py index f951907a..70d6933c 100644 --- a/skills/deeppapernote/scripts/common.py +++ b/skills/deeppapernote/scripts/common.py @@ -65,12 +65,12 @@ def json_input_sha256(value: str | Path) -> str: def emit(payload: dict[str, Any], output_path: str | None = None) -> None: - text = json.dumps(payload, ensure_ascii=False, indent=2) if output_path: ensure_parent(output_path) - Path(output_path).write_text(text + "\n", encoding="utf-8") + Path(output_path).write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") else: - print(text) + # ASCII JSON reads back under any console encoding, such as cp1252 on Windows. + print(json.dumps(payload, indent=2)) def load_json_file(path: str | Path) -> dict[str, Any]: diff --git a/skills/deeppapernote/scripts/run_pipeline.py b/skills/deeppapernote/scripts/run_pipeline.py index ad4f28de..285d5abc 100644 --- a/skills/deeppapernote/scripts/run_pipeline.py +++ b/skills/deeppapernote/scripts/run_pipeline.py @@ -329,6 +329,13 @@ def prepare_steps( ] +def print_outputs(outputs: list[tuple[str, object]]) -> None: + """Print `name=path` lines as UTF-8, so paths survive a console encoding such as cp1252.""" + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8") + print("\n".join(f"{name}={path}" for name, path in outputs)) + + def main() -> None: argv = sys.argv[1:] phase = argv[0] if argv and argv[0] in PHASES else "" @@ -397,7 +404,7 @@ def main() -> None: ("figure_table_decisions", paths["figure_table_decisions"]), ("bundle", paths["bundle"]), ] - print("\n".join(f"{name}={path}" for name, path in outputs)) + print_outputs(outputs) if __name__ == "__main__": diff --git a/skills/deeppapernote/scripts/run_state.py b/skills/deeppapernote/scripts/run_state.py index 2441ea84..fc08b85a 100644 --- a/skills/deeppapernote/scripts/run_state.py +++ b/skills/deeppapernote/scripts/run_state.py @@ -1330,7 +1330,7 @@ def main() -> None: report = inspect_run(args.workdir, args.prefix, bindings) except (OSError, ValueError) as exc: parser.error(str(exc)) - print(json.dumps(report, ensure_ascii=False, indent=2)) + print(json.dumps(report, indent=2)) # ASCII, readable under any console encoding if __name__ == "__main__": diff --git a/skills/deeppapernote/scripts/user_configuration.py b/skills/deeppapernote/scripts/user_configuration.py index 8e09dbfb..8281cfaf 100644 --- a/skills/deeppapernote/scripts/user_configuration.py +++ b/skills/deeppapernote/scripts/user_configuration.py @@ -442,7 +442,7 @@ def main() -> None: result = exc.result except ConfigurationWriteError as exc: result = _blocked_result(user_config_path(), "write_failed", str(exc)) - print(json.dumps(result, ensure_ascii=False, indent=2)) + print(json.dumps(result, indent=2)) # ASCII, readable under any console encoding if result["state"] != "ready": raise SystemExit(2) diff --git a/skills/paper-glossary/scripts/configure_glossary.py b/skills/paper-glossary/scripts/configure_glossary.py index e92cb2e1..d14760f3 100644 --- a/skills/paper-glossary/scripts/configure_glossary.py +++ b/skills/paper-glossary/scripts/configure_glossary.py @@ -36,13 +36,13 @@ def parser() -> argparse.ArgumentParser: def _emit(payload: dict[str, Any], output: str) -> None: - text = json.dumps(payload, ensure_ascii=False, indent=2) + "\n" if output: target = Path(output).expanduser() target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(text, encoding="utf-8") + target.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") else: - print(text, end="") + # ASCII JSON reads back under any console encoding, such as cp1252 on Windows. + print(json.dumps(payload, indent=2)) def main() -> None: diff --git a/skills/paper-glossary/scripts/glossary_common.py b/skills/paper-glossary/scripts/glossary_common.py index d39a94c6..e404e8a5 100644 --- a/skills/paper-glossary/scripts/glossary_common.py +++ b/skills/paper-glossary/scripts/glossary_common.py @@ -340,8 +340,8 @@ def maybe_load_json_record(value: str) -> dict[str, Any] | None: def emit(payload: dict[str, Any], output: str = "") -> None: - text = json.dumps(payload, ensure_ascii=False, indent=2) if output: - Path(output).expanduser().write_text(text + "\n", encoding="utf-8") + Path(output).expanduser().write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") else: - print(text) + # ASCII JSON reads back under any console encoding, such as cp1252 on Windows. + print(json.dumps(payload, indent=2)) diff --git a/skills/research-memory/scripts/aggregate_reading.py b/skills/research-memory/scripts/aggregate_reading.py index 04945d2f..9656de5c 100644 --- a/skills/research-memory/scripts/aggregate_reading.py +++ b/skills/research-memory/scripts/aggregate_reading.py @@ -200,11 +200,11 @@ def main() -> None: else: language = args.language or located["language"] payload = aggregate(Path(located["vault"]), located["papers_dir"], language=language, today=args.today) - text = json.dumps(payload, ensure_ascii=False, indent=2) if args.output: - Path(args.output).write_text(text + "\n", encoding="utf-8") + Path(args.output).write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") else: - sys.stdout.write(text + "\n") + # ASCII JSON reads back under any console encoding, such as cp1252 on Windows. + sys.stdout.write(json.dumps(payload, indent=2) + "\n") if __name__ == "__main__": diff --git a/skills/research-memory/scripts/write_research_memory.py b/skills/research-memory/scripts/write_research_memory.py index 60153a2f..87e393f1 100644 --- a/skills/research-memory/scripts/write_research_memory.py +++ b/skills/research-memory/scripts/write_research_memory.py @@ -121,9 +121,9 @@ def main() -> None: try: result = run(args) except (Refused, ValueError, OSError) as exc: - sys.stdout.write(json.dumps({"status": "refused", "reason": str(exc)}, ensure_ascii=False) + "\n") + sys.stdout.write(json.dumps({"status": "refused", "reason": str(exc)}) + "\n") raise SystemExit(1) from None - sys.stdout.write(json.dumps(result, ensure_ascii=False, indent=2) + "\n") + sys.stdout.write(json.dumps(result, indent=2) + "\n") if __name__ == "__main__": diff --git a/skills/research-memory/tests/test_research_memory_aggregate.py b/skills/research-memory/tests/test_research_memory_aggregate.py index bf75f676..35beba10 100644 --- a/skills/research-memory/tests/test_research_memory_aggregate.py +++ b/skills/research-memory/tests/test_research_memory_aggregate.py @@ -155,3 +155,13 @@ def test_cli_without_a_vault_reports_needs_vault(tmp_path: Path) -> None: result = subprocess.run([sys.executable, str(SCRIPT)], env=env, capture_output=True, text=True) assert result.returncode == 0 assert json.loads(result.stdout)["status"] == "needs_vault" + + +def test_cli_prints_json_a_windows_console_can_read(tmp_path: Path, library: Path) -> None: + env = { + **os.environ, "PYTHONIOENCODING": "cp1252", "PYTHONUTF8": "0", + "DEEPPAPERNOTE_CONFIG_PATH": str(write_config(tmp_path, {"obsidian_vault": str(library), "papers_dir": "Papers"})), + } + result = subprocess.run([sys.executable, str(SCRIPT), "--today", "2026-11-06"], env=env, capture_output=True, text=True) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["paper_count"] == 4 diff --git a/skills/research-memory/tests/test_research_memory_write.py b/skills/research-memory/tests/test_research_memory_write.py index 5de4f095..db437f60 100644 --- a/skills/research-memory/tests/test_research_memory_write.py +++ b/skills/research-memory/tests/test_research_memory_write.py @@ -2,6 +2,7 @@ import hashlib import json +import os import subprocess import sys from pathlib import Path @@ -34,6 +35,8 @@ def write(tmp_path: Path, vault: Path, mode: str, payload: dict, expected: str, [sys.executable, str(SCRIPT), "--vault", str(vault), f"--{mode}", str(data), "--expected-sha256", expected, "--today", "2026-11-06", *extra], capture_output=True, text=True, + # cp1252 stands in for a Windows console, where non-ASCII output fails. + env={**os.environ, "PYTHONIOENCODING": "cp1252", "PYTHONUTF8": "0"}, ) diff --git a/tests/test_console_encoding.py b/tests/test_console_encoding.py new file mode 100644 index 00000000..eaf7c737 --- /dev/null +++ b/tests/test_console_encoding.py @@ -0,0 +1,56 @@ +"""Scripts print to a console whose encoding may not cover Chinese, such as cp1252 on Windows.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = PROJECT_ROOT / "skills" / "deeppapernote" / "scripts" +CP1252 = {**os.environ, "PYTHONIOENCODING": "cp1252", "PYTHONUTF8": "0"} + + +def run_python(code: str, scripts: Path, *args: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-c", code, *args], capture_output=True, text=True, env=CP1252, cwd=scripts, + ) + + +def test_emit_prints_json_a_cp1252_console_can_read(): + result = run_python("import common; common.emit({'domain': '长视频理解'})", SCRIPTS) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout) == {"domain": "长视频理解"} + + +def test_glossary_emit_prints_json_a_cp1252_console_can_read(): + glossary = PROJECT_ROOT / "skills" / "paper-glossary" / "scripts" + for code in ( + "import glossary_common; glossary_common.emit({'term': '注意力'})", + "import configure_glossary; configure_glossary._emit({'term': '注意力'}, '')", + ): + result = run_python(code, glossary) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout) == {"term": "注意力"} + + +def test_run_state_prints_json_a_cp1252_console_can_read(tmp_path): + workdir = tmp_path / "运行目录" + workdir.mkdir() + result = subprocess.run( + [sys.executable, str(SCRIPTS / "run_state.py"), "status", "--workdir", str(workdir), "--prefix", "run"], + capture_output=True, text=True, env=CP1252, + ) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["workdir"] == str(workdir.resolve()) + + +def test_pipeline_paths_print_as_utf8(): + result = subprocess.run( + [sys.executable, "-c", "import run_pipeline; run_pipeline.print_outputs([('workdir', '/tmp/运行')])"], + capture_output=True, env=CP1252, cwd=SCRIPTS, + ) + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert result.stdout.decode("utf-8").strip() == "workdir=/tmp/运行"