deer-flow/tests/skills/test_data_analysis_legacy_xls.py
yi-coding f582b9a87b
fix(skills): read legacy XLS workbooks with an XLS-capable engine (#8215)
* fix(skills): read legacy XLS workbooks with an XLS-capable engine

* fix(skills): keep legacy XLS analysis independent of spatial

* fix(skills): skip unreadable legacy workbooks

* fix(skills): preserve mixed cell types in legacy XLS

---------

Co-authored-by: xiaoyi <xyi74976-del@users.noreply.github.com>
Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
2026-11-14 22:04:58 +08:00

345 lines
11 KiB
Python

"""Read real BIFF workbooks without treating them as OOXML or requiring GDAL."""
import builtins
import datetime
import gzip
import json
import runpy
import shutil
import subprocess
import sys
from pathlib import Path
import pytest
duckdb = pytest.importorskip("duckdb")
openpyxl = pytest.importorskip("openpyxl")
pytest.importorskip("pandas")
pytest.importorskip("xlrd")
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "skills/public/data-analysis/scripts/analyze.py"
FIXTURE = Path(__file__).parent / "fixtures/legacy-xls/workbook.xls"
@pytest.fixture
def analysis():
return runpy.run_path(str(SCRIPT))
def test_real_legacy_xls_keeps_all_sheets_and_cell_types(analysis, tmp_path):
path = tmp_path / "owner's report.xls"
shutil.copyfile(FIXTURE, path)
assert path.read_bytes()[:8] == bytes.fromhex("d0cf11e0a1b11ae1")
table_map = {}
with duckdb.connect() as con:
analysis["_load_excel"](con, str(path), table_map)
assert table_map == {"owner's sales": "owner_s_sales", "Mixed": "Mixed"}
assert con.execute(
"SELECT * FROM owner_s_sales ORDER BY amount DESC"
).fetchall() == [
(7, datetime.datetime(2026, 1, 2), "first", True),
(3, datetime.datetime(2026, 1, 3), None, False),
]
assert con.execute('SELECT value FROM "Mixed"').fetchall() == [
("1",),
("text",),
(None,),
]
def test_legacy_xls_names_share_the_existing_allocator(analysis):
table_map = {}
with duckdb.connect() as con:
for _ in range(2):
analysis["_load_excel"](con, str(FIXTURE), table_map)
assert table_map == {
"owner's sales": "owner_s_sales",
"Mixed": "Mixed",
"owner_s_sales_1": "owner_s_sales_1",
"Mixed_1": "Mixed_1",
}
assert con.execute("SELECT SUM(amount) FROM owner_s_sales_1").fetchone() == (
10,
)
@pytest.mark.parametrize("dependency", ["pandas", "xlrd"])
def test_legacy_xls_missing_dependency_has_setup_instruction(
analysis, monkeypatch, dependency
):
original_import = builtins.__import__
def missing(name, *args, **kwargs):
if name == dependency:
raise ImportError(f"No module named {dependency!r}")
return original_import(name, *args, **kwargs)
monkeypatch.setattr(builtins, "__import__", missing)
with (
duckdb.connect() as con,
pytest.raises(SystemExit, match="python -m pip install pandas xlrd"),
):
analysis["_load_excel"](con, str(FIXTURE), {})
def test_failed_legacy_sheet_does_not_reserve_an_alias(analysis):
table_map = {}
with duckdb.connect() as con:
con.execute("CREATE TABLE owner_s_sales (existing INTEGER)")
analysis["_load_excel"](con, str(FIXTURE), table_map)
assert table_map == {"Mixed": "Mixed"}
assert con.execute("DESCRIBE owner_s_sales").fetchone()[0] == "existing"
def test_legacy_xls_cache_reopens_complete_read_only_tables(
analysis,
tmp_path,
monkeypatch,
):
scope = analysis["main"].__globals__
scope["CACHE_DIR"] = str(tmp_path / "cache")
for _ in range(2):
with analysis["open_analysis_cache"]([str(FIXTURE)]) as (con, table_map):
assert table_map == {"owner's sales": "owner_s_sales", "Mixed": "Mixed"}
output = analysis["action_query"](
con,
'SELECT SUM(amount) AS total FROM "owner\'s sales"',
table_map,
)
assert output.splitlines()[2].strip() == "10"
with pytest.raises(duckdb.Error, match="read-only"):
con.execute("CREATE TABLE forbidden (value INTEGER)")
monkeypatch.setitem(
scope, "load_files", lambda *_: pytest.fail("cache hit must not reload XLS")
)
def test_missing_legacy_dependency_cleans_unpublished_cache(
analysis,
tmp_path,
monkeypatch,
):
scope = analysis["main"].__globals__
cache = tmp_path / "cache"
scope["CACHE_DIR"] = str(cache)
original_import = builtins.__import__
def missing(name, *args, **kwargs):
if name == "xlrd":
raise ImportError("No module named xlrd")
return original_import(name, *args, **kwargs)
monkeypatch.setattr(builtins, "__import__", missing)
with pytest.raises(SystemExit, match="python -m pip install pandas xlrd"):
with analysis["open_analysis_cache"]([str(FIXTURE)]):
pytest.fail("missing XLS engine must not publish a connection")
assert list((cache / "generations-v1").iterdir()) == []
def test_legacy_xls_preserves_text_na_tokens_and_leading_zero_ids(analysis):
table_map = {}
with duckdb.connect() as con:
analysis["_load_excel"](
con, str(FIXTURE.with_name("literal-text.xls")), table_map
)
assert con.execute('SELECT * FROM "Identifiers"').fetchall() == [
("NA", "00123"),
("NULL", "00456"),
("N/A", "00789"),
]
@pytest.fixture
def sparse_mixed_workbook(tmp_path):
path = tmp_path / "sparse mixed.xls"
path.write_bytes(
gzip.decompress(FIXTURE.with_name("sparse-mixed.xls.gz").read_bytes())
)
return str(path)
@pytest.mark.parametrize("sample_size", [17, 1000, 20000])
def test_legacy_xls_infers_entire_mixed_column_and_restores_sampling(
analysis,
sparse_mixed_workbook,
sample_size,
):
table_map = {}
with duckdb.connect() as con:
con.execute("SET pandas_analyze_sample = ?", [sample_size])
analysis["_load_excel"](con, sparse_mixed_workbook, table_map)
assert table_map == {"Mixed": "Mixed"}
assert con.execute(
"SELECT COUNT(*), COUNT(*) FILTER (WHERE value = 'text') FROM \"Mixed\""
).fetchone() == (10001, 1)
assert con.execute(
"SELECT COUNT(*) FROM \"Mixed\" WHERE value = '1'"
).fetchone() == (10000,)
assert con.execute(
"SELECT current_setting('pandas_analyze_sample')"
).fetchone() == (sample_size,)
def test_legacy_xls_restores_sampling_when_table_creation_fails(
analysis,
sparse_mixed_workbook,
):
table_map = {}
with duckdb.connect() as con:
con.execute("SET pandas_analyze_sample = 17")
con.execute('CREATE TABLE "Mixed" (existing INTEGER)')
analysis["_load_excel"](con, sparse_mixed_workbook, table_map)
assert table_map == {}
assert con.execute('DESCRIBE "Mixed"').fetchone()[0] == "existing"
assert con.execute(
"SELECT current_setting('pandas_analyze_sample')"
).fetchone() == (17,)
def _offline_xls_command(tmp_path, files):
extension_dir = tmp_path / "extensions"
extension_dir.mkdir()
repository = tmp_path / "empty-extension-repository"
repository.mkdir()
cache = tmp_path / "cache"
# Configure a real connection in a fresh interpreter. An empty local
# repository makes extension installation fail without any network request;
# all CLI parsing, XLS reads, SQL and cache publication remain production code.
bootstrap = """
import functools
import runpy
import sys
import duckdb
script, extensions, repository, cache, *arguments = sys.argv[1:]
duckdb.connect = functools.partial(
duckdb.connect,
config={
"extension_directory": extensions,
"custom_extension_repository": repository,
"autoinstall_known_extensions": False,
"autoload_known_extensions": False,
},
)
module = runpy.run_path(script)
module["main"].__globals__["CACHE_DIR"] = cache
sys.argv = [script, *arguments]
module["main"]()
"""
return [
sys.executable,
"-c",
bootstrap,
str(SCRIPT),
str(extension_dir),
str(repository),
str(cache),
"--files",
*files,
]
@pytest.mark.parametrize("unreadable_kind", [None, "corrupt", "renamed-xlsx"])
def test_legacy_xls_cli_runs_offline_with_empty_extension_directory(
tmp_path, unreadable_kind
):
"""An XLS-only cold import must not install DuckDB's XLSX reader."""
files = [str(FIXTURE)]
if unreadable_kind:
unreadable = tmp_path / "unreadable.xls"
if unreadable_kind == "corrupt":
unreadable.write_bytes(b"not an Excel workbook")
else:
workbook = openpyxl.Workbook()
workbook.active.append(["value"])
workbook.active.append([99])
workbook.save(unreadable)
workbook.close()
files.insert(0, str(unreadable))
command = _offline_xls_command(tmp_path, files)
inspected = subprocess.run(
[*command, "--action", "inspect"],
cwd=tmp_path,
capture_output=True,
text=True,
timeout=30,
check=False,
)
assert inspected.returncode == 0, inspected.stderr
assert "Traceback" not in inspected.stderr
if unreadable_kind:
assert "Failed to load workbook" in inspected.stderr
assert str(unreadable) in inspected.stderr
assert "owner's sales" in inspected.stdout
assert "Mixed" in inspected.stdout
# A second process reopens the published cache and reads the actual cells.
queried = subprocess.run(
[
*command,
"--action",
"query",
"--sql",
'SELECT SUM(amount) AS total FROM "owner\'s sales"',
],
cwd=tmp_path,
capture_output=True,
text=True,
timeout=30,
check=False,
)
assert queried.returncode == 0, queried.stderr
assert queried.stdout.splitlines()[2].strip() == "10"
assert not list((tmp_path / "extensions").rglob("*"))
def test_legacy_xls_mixed_cell_types_survive_cli_cache(tmp_path):
command = _offline_xls_command(
tmp_path, [str(FIXTURE.with_name("mixed-types.xls"))]
)
output = tmp_path / "result.json"
for cached in (False, True):
result = subprocess.run(
[
*command,
"--action",
"query",
"--sql",
'SELECT *, typeof(amount) AS amount_type FROM "MixedTypes" ORDER BY amount',
"--output-file",
str(output),
],
cwd=tmp_path,
capture_output=True,
text=True,
timeout=30,
check=False,
)
assert result.returncode == 0, result.stderr
assert "Failed to load sheet" not in result.stderr
assert ("Cache hit!" in result.stderr) is cached
assert json.loads(output.read_text(encoding="utf-8")) == [
{
"bool_number": "True",
"date_number": "2026-01-02 00:00:00",
"bool_decimal": "False",
"amount": 1.0,
"amount_type": "DOUBLE",
},
{
"bool_number": "42",
"date_number": "7",
"bool_decimal": "1.25",
"amount": 2.5,
"amount_type": "DOUBLE",
},
{
"bool_number": None,
"date_number": None,
"bool_decimal": None,
"amount": 3.0,
"amount_type": "DOUBLE",
},
]
assert not list((tmp_path / "extensions").rglob("*"))