feat: clean up stale targets removed from config on next build
When a target is present in the state file but no longer in the config, its output file is deleted (or archived if archive_folder is set) and its state entry is removed. This runs at the start of every build.
This commit is contained in:
parent
7503672942
commit
d8e0ed561d
3 changed files with 167 additions and 0 deletions
|
|
@ -44,6 +44,7 @@ class BuildEvent(enum.Enum):
|
||||||
TARGET_FAILED = "failed"
|
TARGET_FAILED = "failed"
|
||||||
TARGET_DEP_FAILED = "dep_failed"
|
TARGET_DEP_FAILED = "dep_failed"
|
||||||
TARGET_NO_PROVIDER = "no_provider"
|
TARGET_NO_PROVIDER = "no_provider"
|
||||||
|
TARGET_REMOVED = "removed"
|
||||||
|
|
||||||
|
|
||||||
ProgressCallback = Callable[[BuildEvent, str, str], None]
|
ProgressCallback = Callable[[BuildEvent, str, str], None]
|
||||||
|
|
@ -184,6 +185,30 @@ async def _build_single_target(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _cleanup_stale_targets(
|
||||||
|
config: ProjectConfig,
|
||||||
|
project_dir: Path,
|
||||||
|
state: BuildState,
|
||||||
|
on_progress: ProgressCallback = _noop_callback,
|
||||||
|
) -> None:
|
||||||
|
"""Remove or archive output files for targets no longer in the config.
|
||||||
|
|
||||||
|
Also removes their entries from the build state.
|
||||||
|
"""
|
||||||
|
stale = [name for name in state.targets if name not in config.targets]
|
||||||
|
for name in stale:
|
||||||
|
output_path = project_dir / name
|
||||||
|
if output_path.exists():
|
||||||
|
if config.archive_folder is not None:
|
||||||
|
dest = archive_file(output_path, project_dir, config.archive_folder)
|
||||||
|
detail = str(dest.relative_to(project_dir)) if dest else ""
|
||||||
|
else:
|
||||||
|
output_path.unlink()
|
||||||
|
detail = "deleted"
|
||||||
|
on_progress(BuildEvent.TARGET_REMOVED, name, detail)
|
||||||
|
del state.targets[name]
|
||||||
|
|
||||||
|
|
||||||
async def run_build(
|
async def run_build(
|
||||||
config: ProjectConfig,
|
config: ProjectConfig,
|
||||||
project_dir: Path,
|
project_dir: Path,
|
||||||
|
|
@ -218,6 +243,13 @@ async def run_build(
|
||||||
graph = get_subgraph_for_target(graph, target)
|
graph = get_subgraph_for_target(graph, target)
|
||||||
|
|
||||||
state = load_state(project_dir, project_name)
|
state = load_state(project_dir, project_name)
|
||||||
|
|
||||||
|
# Clean up targets that are in state but no longer in config.
|
||||||
|
stale_before = set(state.targets)
|
||||||
|
_cleanup_stale_targets(config, project_dir, state, on_progress)
|
||||||
|
if set(state.targets) != stale_before:
|
||||||
|
save_state(state, project_dir, project_name)
|
||||||
|
|
||||||
generations = get_build_order(graph)
|
generations = get_build_order(graph)
|
||||||
target_names = set(config.targets)
|
target_names = set(config.targets)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -143,6 +143,12 @@ def _run_build(
|
||||||
+ click.style(name, bold=True)
|
+ click.style(name, bold=True)
|
||||||
+ f" {detail}"
|
+ f" {detail}"
|
||||||
)
|
)
|
||||||
|
elif event is BuildEvent.TARGET_REMOVED:
|
||||||
|
click.echo(
|
||||||
|
click.style(" rm ", fg="red")
|
||||||
|
+ click.style(name, bold=True)
|
||||||
|
+ click.style(f" ({detail})", dim=True)
|
||||||
|
)
|
||||||
|
|
||||||
start = time.monotonic()
|
start = time.monotonic()
|
||||||
result = asyncio.run(
|
result = asyncio.run(
|
||||||
|
|
|
||||||
|
|
@ -709,6 +709,135 @@ class TestContentTarget:
|
||||||
assert result.failed == {}
|
assert result.failed == {}
|
||||||
|
|
||||||
|
|
||||||
|
class TestStaleTargetCleanup:
|
||||||
|
"""Tests for cleanup of targets removed from config."""
|
||||||
|
|
||||||
|
async def test_stale_target_file_deleted(
|
||||||
|
self, project_dir: Path, write_config: WriteConfig
|
||||||
|
) -> None:
|
||||||
|
config1 = write_config(
|
||||||
|
{
|
||||||
|
"targets": {
|
||||||
|
"a.txt": {"prompt": "first"},
|
||||||
|
"b.txt": {"prompt": "second"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
with patch("hokusai.builder._create_providers", return_value=_fake_providers()):
|
||||||
|
r1 = await run_build(config1, project_dir, _PROJECT)
|
||||||
|
assert set(r1.built) == {"a.txt", "b.txt"}
|
||||||
|
|
||||||
|
# Remove b.txt from config
|
||||||
|
config2 = write_config({"targets": {"a.txt": {"prompt": "first"}}})
|
||||||
|
r2 = await run_build(config2, project_dir, _PROJECT)
|
||||||
|
|
||||||
|
assert not (project_dir / "b.txt").exists()
|
||||||
|
assert "a.txt" in r2.skipped
|
||||||
|
|
||||||
|
async def test_stale_target_archived(
|
||||||
|
self, project_dir: Path, write_config: WriteConfig
|
||||||
|
) -> None:
|
||||||
|
config1 = write_config(
|
||||||
|
{
|
||||||
|
"archive_folder": "archive",
|
||||||
|
"targets": {
|
||||||
|
"a.txt": {"prompt": "first"},
|
||||||
|
"b.txt": {"prompt": "second"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
with patch("hokusai.builder._create_providers", return_value=_fake_providers()):
|
||||||
|
r1 = await run_build(config1, project_dir, _PROJECT)
|
||||||
|
assert set(r1.built) == {"a.txt", "b.txt"}
|
||||||
|
|
||||||
|
config2 = write_config(
|
||||||
|
{
|
||||||
|
"archive_folder": "archive",
|
||||||
|
"targets": {"a.txt": {"prompt": "first"}},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
_ = await run_build(config2, project_dir, _PROJECT)
|
||||||
|
|
||||||
|
assert not (project_dir / "b.txt").exists()
|
||||||
|
assert (project_dir / "archive" / "b.01.txt").exists()
|
||||||
|
|
||||||
|
async def test_stale_state_entry_removed(
|
||||||
|
self, project_dir: Path, write_config: WriteConfig
|
||||||
|
) -> None:
|
||||||
|
config1 = write_config(
|
||||||
|
{
|
||||||
|
"targets": {
|
||||||
|
"a.txt": {"prompt": "first"},
|
||||||
|
"b.txt": {"prompt": "second"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
with patch("hokusai.builder._create_providers", return_value=_fake_providers()):
|
||||||
|
_ = await run_build(config1, project_dir, _PROJECT)
|
||||||
|
|
||||||
|
state = load_state(project_dir, _PROJECT)
|
||||||
|
assert "b.txt" in state.targets
|
||||||
|
|
||||||
|
config2 = write_config({"targets": {"a.txt": {"prompt": "first"}}})
|
||||||
|
_ = await run_build(config2, project_dir, _PROJECT)
|
||||||
|
|
||||||
|
state = load_state(project_dir, _PROJECT)
|
||||||
|
assert "b.txt" not in state.targets
|
||||||
|
assert "a.txt" in state.targets
|
||||||
|
|
||||||
|
async def test_stale_target_file_already_gone(
|
||||||
|
self, project_dir: Path, write_config: WriteConfig
|
||||||
|
) -> None:
|
||||||
|
config1 = write_config(
|
||||||
|
{
|
||||||
|
"targets": {
|
||||||
|
"a.txt": {"prompt": "first"},
|
||||||
|
"b.txt": {"prompt": "second"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
with patch("hokusai.builder._create_providers", return_value=_fake_providers()):
|
||||||
|
_ = await run_build(config1, project_dir, _PROJECT)
|
||||||
|
(project_dir / "b.txt").unlink()
|
||||||
|
|
||||||
|
config2 = write_config({"targets": {"a.txt": {"prompt": "first"}}})
|
||||||
|
_ = await run_build(config2, project_dir, _PROJECT)
|
||||||
|
|
||||||
|
# State entry should still be cleaned even if file is gone
|
||||||
|
state = load_state(project_dir, _PROJECT)
|
||||||
|
assert "b.txt" not in state.targets
|
||||||
|
|
||||||
|
async def test_stale_cleanup_emits_progress(
|
||||||
|
self, project_dir: Path, write_config: WriteConfig
|
||||||
|
) -> None:
|
||||||
|
from hokusai.builder import BuildEvent
|
||||||
|
|
||||||
|
config1 = write_config(
|
||||||
|
{
|
||||||
|
"targets": {
|
||||||
|
"a.txt": {"prompt": "first"},
|
||||||
|
"b.txt": {"prompt": "second"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
events: list[tuple[BuildEvent, str, str]] = []
|
||||||
|
|
||||||
|
def recorder(event: BuildEvent, name: str, detail: str) -> None:
|
||||||
|
events.append((event, name, detail))
|
||||||
|
|
||||||
|
with patch("hokusai.builder._create_providers", return_value=_fake_providers()):
|
||||||
|
_ = await run_build(config1, project_dir, _PROJECT, on_progress=recorder)
|
||||||
|
events.clear()
|
||||||
|
|
||||||
|
config2 = write_config({"targets": {"a.txt": {"prompt": "first"}}})
|
||||||
|
_ = await run_build(config2, project_dir, _PROJECT, on_progress=recorder)
|
||||||
|
|
||||||
|
removed_events = [
|
||||||
|
(e, n) for e, n, _ in events if e is BuildEvent.TARGET_REMOVED
|
||||||
|
]
|
||||||
|
assert ("b.txt",) in [(n,) for _, n in removed_events]
|
||||||
|
|
||||||
|
|
||||||
class TestLoopExpansion:
|
class TestLoopExpansion:
|
||||||
"""End-to-end tests for loop-expanded targets in builds."""
|
"""End-to-end tests for loop-expanded targets in builds."""
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue