Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

draft: redown rewrite #2741

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

draft: redown rewrite #2741

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a24cf32
fix code block args
TheTripleV Sep 13, 2024
8ad2f97
chatgpt rewrote the whole thing into single pass
TheTripleV Sep 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions scripts/doc8_redown.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,9 @@

def new_setattr(self, name, value):
if name == "_raw_content" and value is not None:
value = redown(value.decode("utf-8").replace("\r", "")).encode("utf-8")
# Path(self.filename).with_suffix(".rd").write_text(value.decode("utf-8"))
value = redown(value.decode("utf-8").replace("\r", ""))[0].encode("utf-8")
Path(self.filename).with_suffix(".rd").write_text(value.decode("utf-8"))

old_setattr(self, name, value)


Expand Down
204 changes: 97 additions & 107 deletions source/_extensions/redown.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,10 @@
"""

import re

from pathlib import Path
from sphinx.application import Sphinx
from dataclasses import dataclass


LINK_CORE = r"""
\[ (?P<text>[^\[\]]*?) \] # link brackets + text w/o brackets - allows spaces in text
\(
Expand Down Expand Up @@ -49,118 +47,110 @@
re.VERBOSE, # whitespace and comments are ignored
)


def redown(text: str) -> str:
def redown(text: str):
"""21"""

# replace md code blocks with reST code blocks
"redown, redown, redown, redown"
find = r"(?P<start>^|\n)(?P<btindent> *?)(?P<ticks>```+)(?P<lang>\S+) *?(?:\n\s*?)+(?P<cindent> *?)(?P<code>.*?)(?P=ticks) *?(?P<end>\n|$|\Z)"

def replace(match: re.Match) -> str:
start = match.group("start")
btindent = match.group("btindent")
lang = match.group("lang")
cindent = match.group("cindent")
code = match.group("code")
end = match.group("end")

ret = ""
ret += start
ret += btindent + f".. code-block:: {lang}\n\n"
cindent = 3 * " "

for line in code.splitlines(keepends=True):
if line.strip() == "":
ret += "\n"
"""Transforms markdown-like syntax into reStructuredText and maps input to output line numbers."""
input_lines = text.splitlines(keepends=True)
output_lines = []
line_mapping = [] # List of input line numbers for each output line
i = 0 # Input line index
while i < len(input_lines):
line = input_lines[i]
# Check for code block start
code_block_match = re.match(r'^( *)(`{3,})(\S+)?\s*$', line)
if code_block_match:
# Start of code block
btindent = code_block_match.group(1) or ''
lang = code_block_match.group(3) or ''
is_rst: bool = re.search(r"re?st", lang.lower()) is not None

params_lines = []
params_input_line_numbers = []
code_content_lines = []
code_content_input_line_numbers = []
i += 1
while i < len(input_lines) and not re.match(r'^( *)`{3,}\s*$', input_lines[i]):
code_line = input_lines[i]
if not is_rst and not code_content_lines and re.match(r'^\s*:.*:', code_line):
# Parameter line
params_lines.append(code_line)
params_input_line_numbers.append(i)
else:
# Code content line
code_content_lines.append(code_line)
code_content_input_line_numbers.append(i)
i += 1
if i < len(input_lines):
# Closing ```
i += 1
# Now process the code block
code_block_directive = f'{btindent}.. code-block::{" " + lang if lang else ""}\n'
output_lines.append(code_block_directive)
line_mapping.append(i - len(code_content_lines) - len(params_lines) - 1) # Map to the opening ```
# Add parameter lines immediately after the directive (bug 3)
option_indent = btindent + ' '
for idx, param_line in enumerate(params_lines):
output_lines.append(option_indent + param_line)
line_mapping.append(params_input_line_numbers[idx])
# Add a blank line before the code content
output_lines.append('\n')
if params_input_line_numbers:
line_mapping.append(params_input_line_numbers[-1]) # Map to last param line
else:
ret += cindent + line

return ret

code = lambda: re.sub(find, replace, text, flags=re.DOTALL)
text = code()

# find rst code block ranges
"redown, redown, redown, redown"

@dataclass
class Chunk:
is_code: bool
text: str = ""

chunks: list[Chunk] = []

for line in text.splitlines(keepends=True):
in_code_block = chunks and chunks[-1].is_code
is_code_block_directive = re.match(r"^\s*\.\. code-block::", line)

if not in_code_block and is_code_block_directive:
chunks.append(Chunk(is_code=True))
elif in_code_block:
indent = len(re.match(r"^(\s*)", line).group(1).rstrip("\n"))
code_block_indent = len(
re.match(r"^(\s*)", chunks[-1].text).group(1).rstrip("\n")
)
if len(line.strip()) and indent <= code_block_indent:
if is_code_block_directive:
chunks.append(Chunk(is_code=True))
line_mapping.append(i - len(code_content_lines) - 1) # Map to the opening ```
# Add code content lines
for idx, code_line in enumerate(code_content_lines):
# Handle bug 2: Don't indent empty lines
if code_line.strip():
output_lines.append(' ' + code_line)
else:
chunks.append(Chunk(is_code=False))

if not chunks:
chunks.append(Chunk(is_code=False))
chunks[-1].text += line # existing block

# dont operate on code blocks
for chunk in chunks:
if chunk.is_code:
continue
text = chunk.text

"redown, redown, redown, redown"
heading = lambda prefix, underfix: re.sub(
rf"(^|\n){prefix} +(.+?)(?:$|\n|\Z)",
lambda m: f"{m.group(1)}{m.group(2)}\n{underfix * len(m.group(2))}\n",
text,
)

text = heading("#", "=")
text = heading("##", "-")
text = heading("###", "^")
text = heading("####", "~")

"redown, redown, redown, redown"
role_links = lambda: ROLE_LINK_RE.sub(
lambda m: f"{m.group('role')}`{(t:=m.group('text'))}{' ' if len(t) else ''}<{m.group('link')}>`",
text,
)
text = role_links()

"redown, redown, redown, redown"
links = lambda: LINK_RE.sub(
lambda m: f"`{(t:=m.group('text'))}{' ' if len(t) else ''}<{m.group('link')}>`__",
text,
)
text = links()

"redown, redown, redown, redown"
math = lambda: re.sub(
r"(\b|\s|^)\$([^$\n]+)\$(\b|\s|[^\w]|$)", r"\1:math:`\2`\3", text
)
text = math()

chunk.text = text

text = "".join(chunk.text for chunk in chunks)

return text

output_lines.append('\n')
line_mapping.append(code_content_input_line_numbers[idx])
# Add a blank line after the code content
if code_content_lines:
output_lines.append('\n')
line_mapping.append(code_content_input_line_numbers[idx])
else:
# Process headings
heading_match = re.match(r'^(#+) (.+)$', line)
if heading_match:
level = len(heading_match.group(1))
heading_text = heading_match.group(2).strip()
if level == 1:
underline = '=' * len(heading_text)
elif level == 2:
underline = '-' * len(heading_text)
elif level == 3:
underline = '^' * len(heading_text)
elif level == 4:
underline = '~' * len(heading_text)
else:
underline = '"' * len(heading_text)
output_lines.append(heading_text + '\n')
line_mapping.append(i)
output_lines.append(underline + '\n')
line_mapping.extend([i, i]) # Underline and extra newline map to the same input line
else:
# Process role links, regular links, and inline math
line_processed = ROLE_LINK_RE.sub(
lambda m: f"{m.group('role')}`{(t:=m.group('text'))}{' ' if len(t) else ''}<{m.group('link')}>`",
line)
line_processed = LINK_RE.sub(
lambda m: f"`{(t:=m.group('text'))}{' ' if len(t) else ''}<{m.group('link')}>`__",
line_processed)
line_processed = re.sub(
r"(\b|\s|^)\$([^$\n]+)\$(\b|\s|[^\w]|$)", r"\1:math:`\2`\3", line_processed)
output_lines.append(line_processed)
line_mapping.append(i)
i += 1
output_text = ''.join(output_lines)
return output_text, line_mapping
"redown, redown, redown, redown"

def setup(app: Sphinx):
@(lambda breadcrumb: app.connect("source-read", breadcrumb))
def _(app, docname, content):
content[0] = redown(content[0])
content[0] = redown(content[0])[0]
# Path(app.srcdir, docname).with_suffix(".rd").write_text(content[0], encoding="utf8")

return {
Expand Down
Loading