feat: add support for parsing txt files with only documentation

Author: SuperFola

arkdoc/__main__.py

@@ -64,7 +64,7 @@ def compute(args) -> bool:
                             functions.append(name)
                         else:
                             functions.append(prefix + ":" + name)
-                    elif doc.source == Source.Cpp:
+                    elif doc.source == Source.Cpp or doc.source == Source.Txt:
                         data, _ = extractor({"name": ""}, doc)
                         functions.append(data["name"])
             print("\n".join(functions))

arkdoc/generator/utils.py

@@ -102,11 +102,24 @@ def from_cpp(doc: Documentation) -> spec.Function:
         describe(data, code),
     )
 
+def from_txt(doc: Documentation) -> spec.Function:
+    parameters = {"name": ""}
+    parameters.update(deepcopy(DEFAULT_KEYS))
+    data, code = extractor(parameters, doc)
+
+    return spec.Function(
+        data["name"],
+        f"({data['name']} {' '.join(e.name for e in data['param'])})",
+        describe(data, code),
+    )
+
 
 def documentation_to_specification(doc: Documentation) -> spec.Function:
     if doc.source == Source.ArkScript:
         return from_ark(doc)
     elif doc.source == Source.Cpp:
         return from_cpp(doc)
+    elif doc.source == Source.Txt:
+        return from_txt(doc)
     else:
         raise NotImplementedError

arkdoc/parser/documentation.py

@@ -20,6 +20,7 @@ def deep_flatten(lst):
 class Source(Enum):
     ArkScript = 0
     Cpp = 1
+    Txt = 2
 
 
 @dataclass

arkdoc/parser/parser.py

@@ -4,7 +4,7 @@
 from typing import List
 
 from . import Documentation, Source
-from .tokenizer import cpp_tokenize, tree_from_tokens, tokenize, Token
+from .tokenizer import cpp_tokenize, txt_tokenize, tree_from_tokens, tokenize, Token
 from .. import logger
 
 
@@ -51,6 +51,9 @@ def extract_documentation(self):
         elif self.filename.endswith(".cpp"):
             for node in self.ast:
                 yield Documentation(Source.Cpp, node, None)
+        elif self.filename.endswith(".txt"):
+            for node in self.ast:
+                yield Documentation(Source.Txt, node, None)
         else:
             raise NotImplementedError
 
@@ -67,5 +70,8 @@ def parse(self):
         elif self.filename.endswith(".cpp"):
             self.ast = list(cpp_tokenize(program))
             logger.debug(self.ast)
+        elif self.filename.endswith(".txt"):
+            self.ast = list(txt_tokenize(program))
+            logger.debug(self.ast)
         else:
             logger.error(f"Could not parse {self.filename}")

arkdoc/parser/tokenizer.py

@@ -59,7 +59,7 @@ def tokenize(code: str) -> Generator[Token, Any, None]:
         yield Token(kind, value, line_num, column)
 
 
-def cpp_tokenize(code: str) -> List[str]:
+def cpp_tokenize(code: str) -> Generator[list[Token], Any, None]:
     """
     The following regex accepts this kind of comment
 
@@ -89,6 +89,36 @@ def transform(line: str):
         yield list(transform(value))
 
 
+def txt_tokenize(code: str) -> Generator[list[Token], Any, None]:
+    """
+    The following regex accepts this kind of text blocks
+
+    --#
+    * test
+    * test
+    * test
+    * @brief test
+    #--
+    """
+    tok_regex = r"^ *--#\n(\*( .*)\n)+#--\n"
+    line_num = 1
+    line_start = 0
+
+    def transform(line: str):
+        nonlocal line_num
+        lines = line.split("\n")
+
+        for e in lines[1:]:
+            line = "#" + e.strip()[1:]
+            line_num += 1
+            yield Token("COMMENT", line, line_num, column)
+
+    for mo in re.finditer(tok_regex, code, flags=re.MULTILINE):
+        value = mo.group()
+        column = mo.start() - line_start
+        yield list(transform(value))
+
+
 def tree_from_tokens(tokens: List[Token]) -> List:
     if len(tokens) == 0:
         raise SyntaxError("unexpected EOF")

arkdoc/reader.py

@@ -1,14 +1,22 @@
 #!/usr/bin/env python3
 
 import glob
-from typing import List
+import os.path
+from typing import List, Generator
 
 from .parser import Parser
 
 
-def explore(folder: str) -> List[str]:
-    for ext in ["ark", "cpp"]:
-        yield from glob.glob(f"{folder}/*.{ext}", recursive=True)
+def explore(folder: str) -> Generator[str, None, None]:
+    supported_exts = ["txt", "ark", "cpp"]
+
+    if os.path.isdir(folder):
+        for ext in supported_exts:
+            yield from glob.glob(f"{folder}/*.{ext}", recursive=True)
+    elif os.path.isfile(folder) and os.path.splitext(folder)[1][1:] in supported_exts:
+        yield folder
+    else:
+        raise RuntimeError(f"{folder} isn't a valid file or folder")
 
 
 def parse_all_in(folder: str) -> List[Parser]: