From 1ce0790c0c0b05af20f947ee433df48401d54c50 Mon Sep 17 00:00:00 2001 From: Konstantin Fickel Date: Fri, 30 Jan 2026 18:03:44 +0100 Subject: [PATCH 01/12] feat: add attach_markdown Signed-off-by: Konstantin Fickel --- src/streamer/parse/parse.py | 4 +- test/parse/test_attach_markdown.py | 76 ++++++++++++++++++++++++++++++ test/{ => parse}/test_parse.py | 3 +- 3 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 test/parse/test_attach_markdown.py rename test/{ => parse}/test_parse.py (99%) diff --git a/src/streamer/parse/parse.py b/src/streamer/parse/parse.py index a4c8421..4a8258f 100644 --- a/src/streamer/parse/parse.py +++ b/src/streamer/parse/parse.py @@ -20,8 +20,8 @@ def get_line_number(block_token: BlockToken) -> int: def build_shard( - start_line, - end_line, + start_line: int, + end_line: int, markers: list[str] = [], tags: list[str] = [], children: list[Shard] = [], diff --git a/test/parse/test_attach_markdown.py b/test/parse/test_attach_markdown.py new file mode 100644 index 0000000..21f8a1b --- /dev/null +++ b/test/parse/test_attach_markdown.py @@ -0,0 +1,76 @@ +from faker import Faker + +from streamer.parse import Shard, StreamFile +from streamer.parse.attach_markdown import ( + ShardWithMarkdown, + StreamFileWithMarkdown, + attach_markdown, +) + +fake = Faker() + + +class TestAttachMarkdown: + file_name: str = fake.file_name(extension="md") + + def test_attach_markdown_with_shard(self): + markdown_text = "Hello World\n\nThis is a test." + shard = Shard(start_line=1, end_line=3) + stream_file = StreamFile(filename=self.file_name, shard=shard) + + result = attach_markdown(stream_file, markdown_text) + + assert result == StreamFileWithMarkdown( + filename=self.file_name, + shard=ShardWithMarkdown( + start_line=1, + end_line=3, + markdown_content=markdown_text, + markers=[], + tags=[], + children=[], + ), + ) + + def test_attach_markdown_without_shard(self): + stream_file = StreamFile(filename=self.file_name, shard=None) + + result = attach_markdown(stream_file, "Some markdown text") + + assert result == StreamFileWithMarkdown(filename=self.file_name, shard=None) + + def test_attach_markdown_with_nested_shards(self): + markdown_text = "Header\n\n@Marker1 Content 1\n\n@Marker2 Content 2" + shard = Shard( + start_line=1, + end_line=5, + children=[ + Shard(markers=["Marker1"], start_line=3, end_line=3), + Shard(markers=["Marker2"], start_line=5, end_line=5), + ], + ) + stream_file = StreamFile(filename=self.file_name, shard=shard) + + result = attach_markdown(stream_file, markdown_text) + + assert result.filename == self.file_name + assert result.shard is not None + assert result.shard.start_line == 1 + assert result.shard.end_line == 5 + assert ( + result.shard.markdown_content + == "Header\n\n@Marker1 Content 1\n\n@Marker2 Content 2" + ) + assert len(result.shard.children) == 2 + + # Check first child + assert result.shard.children[0].markers == ["Marker1"] + assert result.shard.children[0].start_line == 3 + assert result.shard.children[0].end_line == 3 + assert result.shard.children[0].markdown_content == "@Marker1 Content 1" + + # Check second child + assert result.shard.children[1].markers == ["Marker2"] + assert result.shard.children[1].start_line == 5 + assert result.shard.children[1].end_line == 5 + assert result.shard.children[1].markdown_content == "@Marker2 Content 2" diff --git a/test/test_parse.py b/test/parse/test_parse.py similarity index 99% rename from test/test_parse.py rename to test/parse/test_parse.py index e9c27ba..0a71671 100644 --- a/test/test_parse.py +++ b/test/parse/test_parse.py @@ -1,6 +1,7 @@ -from streamer.parse import StreamFile, parse_markdown_file, Shard from faker import Faker +from streamer.parse import Shard, StreamFile, parse_markdown_file + fake = Faker() -- 2.53.0 From ee91b2e8db4df7762e285d1a38a5f2a20fe12005 Mon Sep 17 00:00:00 2001 From: Konstantin Fickel Date: Fri, 30 Jan 2026 18:05:26 +0100 Subject: [PATCH 02/12] feat: add all_files iterator to simplify searching Signed-off-by: Konstantin Fickel --- src/streamer/__init__.py | 48 +++++++++++++++------------ src/streamer/parse/attach_markdown.py | 31 +++++++++++++++++ src/streamer/query/task.py | 6 +++- 3 files changed, 62 insertions(+), 23 deletions(-) create mode 100644 src/streamer/parse/attach_markdown.py diff --git a/src/streamer/__init__.py b/src/streamer/__init__.py index c39eede..47fa553 100644 --- a/src/streamer/__init__.py +++ b/src/streamer/__init__.py @@ -1,14 +1,16 @@ import glob import os -import typer +from datetime import datetime +from shutil import move +from typing import Generator + import click +import typer from rich import print from rich.markdown import Markdown from rich.panel import Panel -from shutil import move - -from datetime import datetime +from streamer.parse.attach_markdown import StreamFileWithMarkdown, attach_markdown from streamer.parse.parse import parse_markdown_file from streamer.query.task import find_by_markers from streamer.settings import Settings @@ -16,28 +18,28 @@ from streamer.settings import Settings app = typer.Typer() -@app.command() -def todo() -> None: +def all_files() -> Generator[StreamFileWithMarkdown]: for file_name in glob.glob(f"{glob.escape(Settings().base_folder)}/*.md"): with open(file_name, "r") as file: file_content = file.read() - sharded_document = parse_markdown_file(file_name, file_content) - if sharded_document.shard: - open_tasks = find_by_markers(sharded_document.shard, ["Task"], ["Done"]) + yield attach_markdown( + parse_markdown_file(file_name, file_content), file_content + ) - for task_shard in open_tasks: - print( - Panel( - Markdown( - "\n".join( - file_content.splitlines()[ - task_shard.start_line - 1 : task_shard.end_line - ] - ) - ), - title=f"{file_name}:{task_shard.start_line}", - ) + +@app.command() +def todo() -> None: + for sharded_document in all_files(): + if sharded_document.shard: + open_tasks = find_by_markers(sharded_document.shard, ["Task"], ["Done"]) + + for task_shard in open_tasks: + print( + Panel( + Markdown(task_shard.markdown_content), + title=f"{sharded_document.filename}:{task_shard.start_line}", ) + ) @app.command() @@ -59,7 +61,9 @@ def new() -> None: parsed_content = parse_markdown_file(prelimary_path, content) final_file_name = f"{timestamp}.md" - if parsed_content.shard is not None and len(markers := parsed_content.shard.markers): + if parsed_content.shard is not None and len( + markers := parsed_content.shard.markers + ): final_file_name = f"{timestamp} {' '.join(markers)}.md" final_path = os.path.join(streamer_directory, final_file_name) diff --git a/src/streamer/parse/attach_markdown.py b/src/streamer/parse/attach_markdown.py new file mode 100644 index 0000000..70766d9 --- /dev/null +++ b/src/streamer/parse/attach_markdown.py @@ -0,0 +1,31 @@ +from streamer.parse.shard import Shard, StreamFile + + +class ShardWithMarkdown(Shard): + children: list[ShardWithMarkdown] + markdown_content: str + + +class StreamFileWithMarkdown(StreamFile): + shard: ShardWithMarkdown | None = None # pyright: ignore[reportIncompatibleVariableOverride] + + +def attach_markdown_shard(shard: Shard, markdown_text: str) -> ShardWithMarkdown: + lines = markdown_text.splitlines() + markdown_content = "\n".join(lines[shard.start_line - 1 : shard.end_line]) + return ShardWithMarkdown( + **shard.model_dump(exclude=["children"]), + children=map(lambda child: attach_markdown_shard(child, markdown_text), shard.children), + markdown_content=markdown_content, + ) + + +def attach_markdown(file: StreamFile, markdown_text: str) -> StreamFileWithMarkdown: + if file.shard is None: + return StreamFileWithMarkdown(filename=file.filename, shard=None) + + attached_shard = attach_markdown_shard(file.shard, markdown_text) + return StreamFileWithMarkdown(filename=file.filename, shard=attached_shard) + + +__all__ = ["attach_markdown"] diff --git a/src/streamer/query/task.py b/src/streamer/query/task.py index f4e7cd2..67d3664 100644 --- a/src/streamer/query/task.py +++ b/src/streamer/query/task.py @@ -1,7 +1,11 @@ +from typing import TypeVar + from streamer.parse.shard import Shard +T = TypeVar("T", bound="Shard") -def find_by_markers(shard: Shard, has: list[str], has_not: list[str]) -> list[Shard]: + +def find_by_markers(shard: T, has: list[str], has_not: list[str]) -> list[T]: found_shards = [] if any(tag in has for tag in shard.markers) and not any( -- 2.53.0 From d5b1541436c35289c21fe72f4638b016799c7e99 Mon Sep 17 00:00:00 2001 From: Konstantin Fickel Date: Sat, 31 Jan 2026 17:15:01 +0100 Subject: [PATCH 03/12] feat: extract date & time from tags Signed-off-by: Konstantin Fickel --- src/streamer/localize/extract_datetime.py | 80 +++++++++++- src/streamer/localize/localize.py | 30 +++-- src/streamer/localize/localized_shard.py | 4 + src/streamer/parse/attach_markdown.py | 8 +- test/localize/test_extract_datetime.py | 141 ++++++++++++++++++++-- test/test_localize.py | 5 +- 6 files changed, 246 insertions(+), 22 deletions(-) diff --git a/src/streamer/localize/extract_datetime.py b/src/streamer/localize/extract_datetime.py index ed82955..df15935 100644 --- a/src/streamer/localize/extract_datetime.py +++ b/src/streamer/localize/extract_datetime.py @@ -1,9 +1,9 @@ -from datetime import datetime -import re import os +import re +from datetime import date, datetime, time -def extract_date_from_file_name(file_name: str) -> datetime | None: +def extract_datetime_from_file_name(file_name: str) -> datetime | None: FILE_NAME_REGEX = r"^(?P\d{8})(?:-(?P