feat: clean up stale targets removed from config on next build
When a target is present in the state file but no longer in the config, its output file is deleted (or archived if archive_folder is set) and its state entry is removed. This runs at the start of every build.
This commit is contained in:
parent
7503672942
commit
d8e0ed561d
3 changed files with 167 additions and 0 deletions
|
|
@ -44,6 +44,7 @@ class BuildEvent(enum.Enum):
|
|||
TARGET_FAILED = "failed"
|
||||
TARGET_DEP_FAILED = "dep_failed"
|
||||
TARGET_NO_PROVIDER = "no_provider"
|
||||
TARGET_REMOVED = "removed"
|
||||
|
||||
|
||||
ProgressCallback = Callable[[BuildEvent, str, str], None]
|
||||
|
|
@ -184,6 +185,30 @@ async def _build_single_target(
|
|||
)
|
||||
|
||||
|
||||
def _cleanup_stale_targets(
|
||||
config: ProjectConfig,
|
||||
project_dir: Path,
|
||||
state: BuildState,
|
||||
on_progress: ProgressCallback = _noop_callback,
|
||||
) -> None:
|
||||
"""Remove or archive output files for targets no longer in the config.
|
||||
|
||||
Also removes their entries from the build state.
|
||||
"""
|
||||
stale = [name for name in state.targets if name not in config.targets]
|
||||
for name in stale:
|
||||
output_path = project_dir / name
|
||||
if output_path.exists():
|
||||
if config.archive_folder is not None:
|
||||
dest = archive_file(output_path, project_dir, config.archive_folder)
|
||||
detail = str(dest.relative_to(project_dir)) if dest else ""
|
||||
else:
|
||||
output_path.unlink()
|
||||
detail = "deleted"
|
||||
on_progress(BuildEvent.TARGET_REMOVED, name, detail)
|
||||
del state.targets[name]
|
||||
|
||||
|
||||
async def run_build(
|
||||
config: ProjectConfig,
|
||||
project_dir: Path,
|
||||
|
|
@ -218,6 +243,13 @@ async def run_build(
|
|||
graph = get_subgraph_for_target(graph, target)
|
||||
|
||||
state = load_state(project_dir, project_name)
|
||||
|
||||
# Clean up targets that are in state but no longer in config.
|
||||
stale_before = set(state.targets)
|
||||
_cleanup_stale_targets(config, project_dir, state, on_progress)
|
||||
if set(state.targets) != stale_before:
|
||||
save_state(state, project_dir, project_name)
|
||||
|
||||
generations = get_build_order(graph)
|
||||
target_names = set(config.targets)
|
||||
|
||||
|
|
|
|||
|
|
@ -143,6 +143,12 @@ def _run_build(
|
|||
+ click.style(name, bold=True)
|
||||
+ f" {detail}"
|
||||
)
|
||||
elif event is BuildEvent.TARGET_REMOVED:
|
||||
click.echo(
|
||||
click.style(" rm ", fg="red")
|
||||
+ click.style(name, bold=True)
|
||||
+ click.style(f" ({detail})", dim=True)
|
||||
)
|
||||
|
||||
start = time.monotonic()
|
||||
result = asyncio.run(
|
||||
|
|
|
|||
|
|
@ -709,6 +709,135 @@ class TestContentTarget:
|
|||
assert result.failed == {}
|
||||
|
||||
|
||||
class TestStaleTargetCleanup:
|
||||
"""Tests for cleanup of targets removed from config."""
|
||||
|
||||
async def test_stale_target_file_deleted(
|
||||
self, project_dir: Path, write_config: WriteConfig
|
||||
) -> None:
|
||||
config1 = write_config(
|
||||
{
|
||||
"targets": {
|
||||
"a.txt": {"prompt": "first"},
|
||||
"b.txt": {"prompt": "second"},
|
||||
}
|
||||
}
|
||||
)
|
||||
with patch("hokusai.builder._create_providers", return_value=_fake_providers()):
|
||||
r1 = await run_build(config1, project_dir, _PROJECT)
|
||||
assert set(r1.built) == {"a.txt", "b.txt"}
|
||||
|
||||
# Remove b.txt from config
|
||||
config2 = write_config({"targets": {"a.txt": {"prompt": "first"}}})
|
||||
r2 = await run_build(config2, project_dir, _PROJECT)
|
||||
|
||||
assert not (project_dir / "b.txt").exists()
|
||||
assert "a.txt" in r2.skipped
|
||||
|
||||
async def test_stale_target_archived(
|
||||
self, project_dir: Path, write_config: WriteConfig
|
||||
) -> None:
|
||||
config1 = write_config(
|
||||
{
|
||||
"archive_folder": "archive",
|
||||
"targets": {
|
||||
"a.txt": {"prompt": "first"},
|
||||
"b.txt": {"prompt": "second"},
|
||||
},
|
||||
}
|
||||
)
|
||||
with patch("hokusai.builder._create_providers", return_value=_fake_providers()):
|
||||
r1 = await run_build(config1, project_dir, _PROJECT)
|
||||
assert set(r1.built) == {"a.txt", "b.txt"}
|
||||
|
||||
config2 = write_config(
|
||||
{
|
||||
"archive_folder": "archive",
|
||||
"targets": {"a.txt": {"prompt": "first"}},
|
||||
}
|
||||
)
|
||||
_ = await run_build(config2, project_dir, _PROJECT)
|
||||
|
||||
assert not (project_dir / "b.txt").exists()
|
||||
assert (project_dir / "archive" / "b.01.txt").exists()
|
||||
|
||||
async def test_stale_state_entry_removed(
|
||||
self, project_dir: Path, write_config: WriteConfig
|
||||
) -> None:
|
||||
config1 = write_config(
|
||||
{
|
||||
"targets": {
|
||||
"a.txt": {"prompt": "first"},
|
||||
"b.txt": {"prompt": "second"},
|
||||
}
|
||||
}
|
||||
)
|
||||
with patch("hokusai.builder._create_providers", return_value=_fake_providers()):
|
||||
_ = await run_build(config1, project_dir, _PROJECT)
|
||||
|
||||
state = load_state(project_dir, _PROJECT)
|
||||
assert "b.txt" in state.targets
|
||||
|
||||
config2 = write_config({"targets": {"a.txt": {"prompt": "first"}}})
|
||||
_ = await run_build(config2, project_dir, _PROJECT)
|
||||
|
||||
state = load_state(project_dir, _PROJECT)
|
||||
assert "b.txt" not in state.targets
|
||||
assert "a.txt" in state.targets
|
||||
|
||||
async def test_stale_target_file_already_gone(
|
||||
self, project_dir: Path, write_config: WriteConfig
|
||||
) -> None:
|
||||
config1 = write_config(
|
||||
{
|
||||
"targets": {
|
||||
"a.txt": {"prompt": "first"},
|
||||
"b.txt": {"prompt": "second"},
|
||||
}
|
||||
}
|
||||
)
|
||||
with patch("hokusai.builder._create_providers", return_value=_fake_providers()):
|
||||
_ = await run_build(config1, project_dir, _PROJECT)
|
||||
(project_dir / "b.txt").unlink()
|
||||
|
||||
config2 = write_config({"targets": {"a.txt": {"prompt": "first"}}})
|
||||
_ = await run_build(config2, project_dir, _PROJECT)
|
||||
|
||||
# State entry should still be cleaned even if file is gone
|
||||
state = load_state(project_dir, _PROJECT)
|
||||
assert "b.txt" not in state.targets
|
||||
|
||||
async def test_stale_cleanup_emits_progress(
|
||||
self, project_dir: Path, write_config: WriteConfig
|
||||
) -> None:
|
||||
from hokusai.builder import BuildEvent
|
||||
|
||||
config1 = write_config(
|
||||
{
|
||||
"targets": {
|
||||
"a.txt": {"prompt": "first"},
|
||||
"b.txt": {"prompt": "second"},
|
||||
}
|
||||
}
|
||||
)
|
||||
events: list[tuple[BuildEvent, str, str]] = []
|
||||
|
||||
def recorder(event: BuildEvent, name: str, detail: str) -> None:
|
||||
events.append((event, name, detail))
|
||||
|
||||
with patch("hokusai.builder._create_providers", return_value=_fake_providers()):
|
||||
_ = await run_build(config1, project_dir, _PROJECT, on_progress=recorder)
|
||||
events.clear()
|
||||
|
||||
config2 = write_config({"targets": {"a.txt": {"prompt": "first"}}})
|
||||
_ = await run_build(config2, project_dir, _PROJECT, on_progress=recorder)
|
||||
|
||||
removed_events = [
|
||||
(e, n) for e, n, _ in events if e is BuildEvent.TARGET_REMOVED
|
||||
]
|
||||
assert ("b.txt",) in [(n,) for _, n in removed_events]
|
||||
|
||||
|
||||
class TestLoopExpansion:
|
||||
"""End-to-end tests for loop-expanded targets in builds."""
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue