[kernel][asmxygen.py] Implement macro arguments parsing

git-svn-id: svn://kolibrios.org@8855 a494cfbc-eb01-0410-851d-a64ba20cac60
This commit is contained in:
Magomed Kostoev (mkostoevr) 2021-06-15 08:58:03 +00:00
parent 0cb57998a6
commit b84899dbd5

View File

@ -11,21 +11,28 @@ clean_generated_stuff = False
dump_symbols = False
# Print symbol stats: --stats
print_stats = False
# Do not write warnings file: --nowarn
enable_warnings = True
# Constants
link_root = "http://websvn.kolibrios.org/filedetails.php?repname=Kolibri+OS&path=/kernel/trunk"
# Warning list
warnings = ""
# Parse arguments
parser = argparse.ArgumentParser()
parser.add_argument("-o", help="Doxygen output folder")
parser.add_argument("--clean", help="Remove generated files", action="store_true")
parser.add_argument("--dump", help="Dump all defined symbols", action="store_true")
parser.add_argument("--stats", help="Print symbol stats", action="store_true")
parser.add_argument("--nowarn", help="Do not write warnings file", action="store_true")
args = parser.parse_args()
doxygen_src_path = args.o if args.o else 'docs/doxygen'
clean_generated_stuff = args.clean
dump_symbols = args.dump
print_stats = args.stats
enable_warnings = not args.nowarn
# kernel_structure["filename"] = {
# [ [], # [0] Variables - [ line, name ]
@ -40,11 +47,239 @@ LABELS = 3
STRUCTURES = 4
kernel_structure = {}
class AsmVariable:
def __init__(self, line, name, type, init, comment, line_span):
self.line = line
self.name = name
self.type = type
self.init = init
self.comment = comment # Comment after the definition (a dd 0 ; Comment)
self.line_span = line_span # How much .asm lines its definition takes
class AsmFunction:
def __init__(self, line, name):
self.line = line
self.name = name
class AsmLabel:
def __init__(self, line, name):
self.line = line
self.name = name
class AsmMacro:
def __init__(self, line, name, comment, args):
self.line = line
self.name = name
self.comment = comment
self.args = args
class AsmStruct:
def __init__(self, line, name):
self.line = line
self.name = name
def parse_variable(asm_file_name, lines, line_idx):
global warnings
def curr():
try: return line[i]
except: return ''
# Returns current and then increments current index
def step():
nonlocal i
c = curr()
i += 1
return c
line = lines[line_idx]
i = 0
# Skip first spaces
while curr().isspace(): step()
# Get name
name = ""
while curr().isalnum() or curr() == '_' or curr() == '.': name += step()
# Skip spaces after variable name
while curr().isspace(): step()
# Get type specifier (db, dd, etc.)
type = ""
while curr().isalnum() or curr() == '_': type += step()
# Skip spaces after type specifier
while curr().isspace(): step()
# Get initial value (everything up to end of the line or comment)
init = ""
while curr() and curr() != ';': init += step()
# Get comment
comment = ""
if curr() == ';':
step() # Skip ';'
while curr(): comment += step()
# Process type
if type == "db": type = "byte"
elif type == "dw": type = "word"
elif type == "dd": type = "dword"
elif type == "dq": type = "qword"
else: raise Exception(f"Unexpected type: '{type}' (i = {i})")
# Process comment
if comment == "": comment = "Undocumented"
else:
comment = comment.lstrip()
if (len(comment) == 0):
comment = "!!! EMPTY_COMMENT"
warnings += f"{asm_file_name}:{line_idx + 1}: Empty comment in\n"
if comment[0].islower():
warnings += f"{asm_file_name}:{line_idx + 1}: Сomment sarting with lowercase\n"
# Build the result
result = AsmVariable(line_idx + 1, name, type, init, comment, 1)
return (1, result)
def is_id(c):
return c.isprintable() and c not in "+-/*=<>()[]{}:,|&~#`'\" \n\r\t\v"
def get_comment_begin(line):
result = len(line)
in_str = False
for i in range(len(line)):
if in_str:
if line[i] == in_str: in_str = False
i += 1
elif line[i] == '\'' or line[i] == '\"':
in_str = line[i]
i += 1
elif line[i] == ';':
result = i
break
else:
i += 1
return result
def get_comment(line):
return line[get_comment_begin(line):]
def remove_comment(line):
return line[0:get_comment_begin(line)]
def insert_comment(line, comment):
comment_begin = get_comment_begin(line)
line_left = line[:get_comment_begin(line)]
line_right = line[get_comment_begin(line):]
return line_left + comment + line_right
def has_line_wrap(line):
if remove_comment(line).rstrip()[-1] == '\\':
return True
return False
def remove_line_wrap(line):
if remove_comment(line).rstrip()[-1] == '\\':
return remove_comment(line).rstrip()[:-1]
return line
def parse_macro(asm_file_name, lines, line_idx):
line_idx_orig = line_idx
global warnings
def curr():
try: return line[i]
except: return ''
# Returns current and then increments current index
def step():
nonlocal i
c = curr()
i += 1
return c
line = lines[line_idx]
# Handle line wraps ('\' at the end)
while has_line_wrap(line):
next_line = lines[line_idx + 1]
prev_line_comment = get_comment(line)
line = remove_line_wrap(line) + insert_comment(next_line, prev_line_comment)
line_idx += 1
i = 0
# Skip first spaces
while curr().isspace(): step()
# Read "macro" keyword
keyword = ""
while is_id(curr()): keyword += step()
if keyword != "macro": raise Exception(f"Not a macro: {line}")
# Skip spaces after "macro"
while curr().isspace(): step()
# Read macro name
name = ""
while curr() and not curr().isspace(): name += step()
# Skip spaces after macro name
while curr().isspace(): step()
# Find all arguments
args = []
arg = ''
while curr() and curr() != ';' and curr() != '{':
# Collect identifier
if is_id(curr()):
arg += step()
# Save the collected identifier
elif curr() == ',':
args.append(arg)
arg = ''
step()
# Just push the '['
elif curr() == '[':
args.append(step())
# Just push the identifier and get ']' ready to be pushed on next comma
elif curr() == ']':
args.append(arg)
arg = step()
# Just push the identifier and get '*' ready to be pushed on next comma
elif curr() == '*':
args.append(arg)
arg = step()
# Just skip whitespaces
elif curr().isspace():
step()
# Something unexpected
else:
raise Exception(f"Unexpected symbol '{curr()}' at index #{i} " +
f"in the macro declaration:\n'{line}'")
if arg != '':
args.append(arg)
if len(args) > 0:
print(line, args)
# Find a comment if any
comment = ""
while curr() and curr() != ';': step()
if curr() == ';':
step()
while curr(): comment += step()
# Find end of the macro
end_of_macro = False
while not end_of_macro:
line = lines[line_idx]
rbraces = re.finditer('}', line)
for rbrace_match in rbraces:
rbrace_idx = rbrace_match.start()
if line[rbrace_idx - 1] != '\\':
end_of_macro = True
line_idx += 1
# Process comment
if comment != "":
comment = comment.lstrip()
if (len(comment) == 0):
comment = "!!! EMPTY_COMMENT"
warnings += f"{asm_file_name}:{line_idx + 1}: Empty comment in\n"
if comment[0].islower():
warnings += f"{asm_file_name}:{line_idx + 1}: Сomment sarting with lowercase\n"
# Build the output
line_span = line_idx - line_idx_orig + 1
result = AsmMacro(line_idx_orig, name, comment, args)
return (line_span, result)
def get_declarations(asm_file_contents, asm_file_name):
asm_file_name = asm_file_name.replace("./", "")
kernel_structure[asm_file_name] = [ [], [], [], [], [] ]
variable_pattern = re.compile(r'^\s*([\w\.]+)\s+d([bwdq])\s+([^;]*)\s*([;].*)?')
variable_pattern = re.compile(r'^\s*[\w\.]+\s+d[bwdq]\s+.*')
macro_pattern = re.compile(r'^\s*macro\s+([\w]+).*')
proc_pattern = re.compile(r'^\s*proc\s+([\w\.]+).*')
label_pattern = re.compile(r'^(?!;)\s*([\w\.]+):.*')
@ -55,43 +290,23 @@ def get_declarations(asm_file_contents, asm_file_name):
while line_idx < len(lines):
line = lines[line_idx]
match = variable_pattern.findall(line)
if len(match) > 0:
(var_name, var_type, var_init, var_comm) = match[0]
if var_comm == "":
var_comm = "Undocumented"
else:
var_comm = var_comm[1:].lstrip()
if (len(var_comm) == 0):
var_comm = "!!! EMPTY_COMMENT"
if var_comm[0].islower(): var_comm = "!!! LOWERCASE COMMENT " + var_comm
if var_type == "b": var_type = "byte"
if var_type == "w": var_type = "word"
if var_type == "d": var_type = "dword"
if var_type == "q": var_type = "qword"
kernel_structure[asm_file_name][VARIABLES].append([ line_idx + 1, var_name, var_type, var_init, var_comm ])
line_idx += 1
if variable_pattern.match(line):
(skip_lines, var) = parse_variable(asm_file_name, lines, line_idx)
kernel_structure[asm_file_name][VARIABLES].append(var)
line_idx += skip_lines
continue
match = macro_pattern.findall(line)
if len(match) > 0:
macro_name = match[0]
kernel_structure[asm_file_name][MACROS].append([ line_idx + 1, macro_name ])
end_of_macro = False
while not end_of_macro:
line = lines[line_idx]
rbraces = re.finditer('}', line)
for rbrace_match in rbraces:
rbrace_idx = rbrace_match.start()
if line[rbrace_idx - 1] != '\\':
end_of_macro = True
line_idx += 1
(skip_lines, macro) = parse_macro(asm_file_name, lines, line_idx)
kernel_structure[asm_file_name][MACROS].append(macro)
line_idx += skip_lines
continue
match = proc_pattern.findall(line)
if len(match) > 0:
proc_name = match[0]
kernel_structure[asm_file_name][PROCEDURES].append([ line_idx + 1, proc_name ])
kernel_structure[asm_file_name][PROCEDURES].append(AsmFunction(line_idx + 1, proc_name))
line_idx += 1
continue
@ -100,14 +315,14 @@ def get_declarations(asm_file_contents, asm_file_name):
label_name = match[0]
# Don't count local labels
if label_name[0] != '.':
kernel_structure[asm_file_name][LABELS].append([ line_idx + 1, label_name ])
kernel_structure[asm_file_name][LABELS].append(AsmLabel(line_idx + 1, label_name))
line_idx += 1
continue
match = struct_pattern.findall(line)
if len(match) > 0:
struct_name = match[0]
kernel_structure[asm_file_name][STRUCTURES].append([ line_idx + 1, struct_name ])
kernel_structure[asm_file_name][STRUCTURES].append(AsmStruct(line_idx + 1, struct_name))
end_of_struct = False
while not end_of_struct:
line = lines[line_idx]
@ -149,23 +364,23 @@ if dump_symbols:
if len(kernel_structure[source][VARIABLES]) > 0:
print(" Variables:")
for variable in kernel_structure[source][VARIABLES]:
print(f" {variable[0]}: {variable[1]}")
print(f" {variable.line}: {variable.name}")
if len(kernel_structure[source][PROCEDURES]) > 0:
print(" Procedures:")
for procedure in kernel_structure[source][PROCEDURES]:
print(f" {procedure[0]}: {procedure[1]}")
print(f" {procedure.line}: {procedure.name}")
if len(kernel_structure[source][LABELS]) > 0:
print(" Global labels:")
for label in kernel_structure[source][LABELS]:
print(f" {label[0]}: {label[1]}")
print(f" {label.line}: {label.name}")
if len(kernel_structure[source][MACROS]) > 0:
print(" Macroses:")
for macro in kernel_structure[source][MACROS]:
print(f" {macro[0]}: {macro[1]}")
print(f" {macro.line}: {macro.name}")
if len(kernel_structure[source][STRUCTURES]) > 0:
print(" Structures:")
for struct in kernel_structure[source][STRUCTURES]:
print(f" {struct[0]}: {struct[1]}")
print(f" {struct.line}: {struct.name}")
if print_stats:
# Collect stats
@ -208,8 +423,12 @@ def write_something(source, somehing):
f.write(somehing)
f.close()
def write_variable(source, line, name, type, init, brief):
name = name.replace(".", "_")
def write_variable(source, variable):
line = variable.line
type = variable.type
init = variable.init
brief = variable.comment
name = variable.name.replace(".", "_")
something = (f"/**\n" +
f" * @brief {brief}\n" +
f" * @par Initial value\n" +
@ -240,14 +459,31 @@ def write_label(source, line, name, brief = "Undocumented"):
f"void {name}();\n\n")
write_something(source, something)
def write_macro(source, line, name, brief = "Undocumented"):
name = name.replace(".", "_")
def write_macro(source, macro):
if macro.comment == "": brief = "Undocumented"
else: brief = macro.comment
line = macro.line
name = macro.name.replace(".", "_").replace("@", "_")
# Construct arg list without '['s, ']'s and '*'s
args = [arg for arg in macro.args if arg not in "[]*"]
# Construct C-like arg list
arg_list = ""
if len(args) > 0:
arg_list += '('
argc = 0
for arg in args:
if argc != 0:
arg_list += ", "
arg_list += arg
argc += 1
arg_list += ')'
something = (f"/**\n" +
f" * @def {name}\n" +
f" * @brief {brief}\n" +
f" * @par Source\n" +
f" * <a href='{link_root}/{source}#line-{line}'>{source}:{line}</a>\n" +
f" */\n#define {name}\n\n")
f" */\n#define {name}{arg_list}\n\n")
write_something(source, something)
def write_structure(source, line, name, brief = "Undocumented"):
@ -267,17 +503,20 @@ for source in kernel_structure:
# Write variables doxygen of the source file
if len(kernel_structure[source][VARIABLES]) > 0:
for variable in kernel_structure[source][VARIABLES]:
write_variable(source, variable[0], variable[1], variable[2], variable[3], variable[4])
write_variable(source, variable)
if len(kernel_structure[source][PROCEDURES]) > 0:
for procedure in kernel_structure[source][PROCEDURES]:
write_procedure(source, procedure[0], procedure[1])
write_procedure(source, procedure.line, procedure.name)
if len(kernel_structure[source][LABELS]) > 0:
for label in kernel_structure[source][LABELS]:
write_label(source, label[0], label[1])
write_label(source, label.line, label.name)
if len(kernel_structure[source][MACROS]) > 0:
for macro in kernel_structure[source][MACROS]:
write_macro(source, macro[0], macro[1])
write_macro(source, macro)
if len(kernel_structure[source][STRUCTURES]) > 0:
for structure in kernel_structure[source][STRUCTURES]:
write_structure(source, structure[0], structure[1])
write_structure(source, structure.line, structure.name)
i += 1
if enable_warnings:
open('asmxygen.txt', "w", encoding = "utf-8").write(warnings)