chore: refactor and simplify parsing

Signed-off-by: Konstantin Fickel <mail@konstantinfickel.de>

src/streamer/parse.py

@@ -91,6 +91,9 @@ def split_at(list_to_be_split: list[T], positions: list[int]):
 def to_shard(
     tokens: list[Token], start_line: int, end_line: int, children: list[Shard] = []
 ) -> Shard:
+    if len(children) == 1 and len(tokens) == 0:
+        return children[0]
+
     markers, tags = extract_markers_and_tags(tokens) if len(tokens) > 0 else ([], [])
 
     return Shard(
@@ -102,13 +105,8 @@ def to_shard(
     )
 
 
-def parse_markdown_file(file_name: str, file_content: str) -> StreamFile:
-    shard = None
-    with TagMarkdownRenderer():
-        ast = Document(file_content)
-        line_count = len(file_content.splitlines())
-
-        if block_tokens := ast.children:
+def parse_paragraph_shards(block_tokens: list[BlockToken], end_line: int) -> Shard:
+    start_line = block_tokens[0].line_number
     shard_starts = find_shard_positions(block_tokens)
 
     child_shards: list[Shard] = []
@@ -117,19 +115,26 @@ def parse_markdown_file(file_name: str, file_content: str) -> StreamFile:
     for i in range(len(block_tokens)):
         token = block_tokens[i]
         if i in shard_starts:
-                    end_line = (
+            shard_end_line = (
                 block_tokens[i + 1].line_number - 1
                 if i + 1 < len(block_tokens)
-                        else line_count
+                else end_line
             )
-                    child_shards.append(to_shard([token], token.line_number, end_line))
+            child_shards.append(to_shard([token], token.line_number, shard_end_line))
         else:
             own_elements.append(token)
 
-            if len(child_shards) == 1 and len(own_elements) == 0:
-                shard = child_shards[0]
-            else:
-                shard = to_shard(own_elements, 1, line_count, children=child_shards)
+    return to_shard(own_elements, start_line, end_line, children=child_shards)
+
+
+def parse_markdown_file(file_name: str, file_content: str) -> StreamFile:
+    shard = None
+    with TagMarkdownRenderer():
+        ast = Document(file_content)
+        line_count = len(file_content.splitlines())
+
+        if block_tokens := ast.children:
+            shard = parse_paragraph_shards(block_tokens, line_count)
 
     return StreamFile(shard=shard, filename=file_name)
 

test/test_parse.py

@@ -26,14 +26,14 @@ class TestParseProcess:
         )
 
     def test_parse_basic_multi_line_file(self):
-        test_file = "Hello World\nHello again!"
+        test_file = "Hello World\n\nHello again!"
         assert parse_markdown_file(self.file_name, test_file) == StreamFile(
             filename=self.file_name,
             shard=Shard(
                 markers=[],
                 tags=[],
                 start_line=1,
-                end_line=2,
+                end_line=3,
                 children=[],
             ),
         )