From b84899dbd5ee631fcf7b5d6366dc5438ae9f8e69 Mon Sep 17 00:00:00 2001 From: "Magomed Kostoev (mkostoevr)" Date: Tue, 15 Jun 2021 08:58:03 +0000 Subject: [PATCH] [kernel][asmxygen.py] Implement macro arguments parsing git-svn-id: svn://kolibrios.org@8855 a494cfbc-eb01-0410-851d-a64ba20cac60 --- kernel/trunk/asmxygen.py | 331 +++++++++++++++++++++++++++++++++------ 1 file changed, 285 insertions(+), 46 deletions(-) diff --git a/kernel/trunk/asmxygen.py b/kernel/trunk/asmxygen.py index 8710b3a5b9..562ec16dd4 100644 --- a/kernel/trunk/asmxygen.py +++ b/kernel/trunk/asmxygen.py @@ -11,21 +11,28 @@ clean_generated_stuff = False dump_symbols = False # Print symbol stats: --stats print_stats = False +# Do not write warnings file: --nowarn +enable_warnings = True # Constants link_root = "http://websvn.kolibrios.org/filedetails.php?repname=Kolibri+OS&path=/kernel/trunk" +# Warning list +warnings = "" + # Parse arguments parser = argparse.ArgumentParser() parser.add_argument("-o", help="Doxygen output folder") parser.add_argument("--clean", help="Remove generated files", action="store_true") parser.add_argument("--dump", help="Dump all defined symbols", action="store_true") parser.add_argument("--stats", help="Print symbol stats", action="store_true") +parser.add_argument("--nowarn", help="Do not write warnings file", action="store_true") args = parser.parse_args() doxygen_src_path = args.o if args.o else 'docs/doxygen' clean_generated_stuff = args.clean dump_symbols = args.dump print_stats = args.stats +enable_warnings = not args.nowarn # kernel_structure["filename"] = { # [ [], # [0] Variables - [ line, name ] @@ -40,11 +47,239 @@ LABELS = 3 STRUCTURES = 4 kernel_structure = {} +class AsmVariable: + def __init__(self, line, name, type, init, comment, line_span): + self.line = line + self.name = name + self.type = type + self.init = init + self.comment = comment # Comment after the definition (a dd 0 ; Comment) + self.line_span = line_span # How much .asm lines its definition takes + +class AsmFunction: + def __init__(self, line, name): + self.line = line + self.name = name + +class AsmLabel: + def __init__(self, line, name): + self.line = line + self.name = name + +class AsmMacro: + def __init__(self, line, name, comment, args): + self.line = line + self.name = name + self.comment = comment + self.args = args + +class AsmStruct: + def __init__(self, line, name): + self.line = line + self.name = name + +def parse_variable(asm_file_name, lines, line_idx): + global warnings + + def curr(): + try: return line[i] + except: return '' + + # Returns current and then increments current index + def step(): + nonlocal i + c = curr() + i += 1 + return c + + line = lines[line_idx] + i = 0 + # Skip first spaces + while curr().isspace(): step() + # Get name + name = "" + while curr().isalnum() or curr() == '_' or curr() == '.': name += step() + # Skip spaces after variable name + while curr().isspace(): step() + # Get type specifier (db, dd, etc.) + type = "" + while curr().isalnum() or curr() == '_': type += step() + # Skip spaces after type specifier + while curr().isspace(): step() + # Get initial value (everything up to end of the line or comment) + init = "" + while curr() and curr() != ';': init += step() + # Get comment + comment = "" + if curr() == ';': + step() # Skip ';' + while curr(): comment += step() + # Process type + if type == "db": type = "byte" + elif type == "dw": type = "word" + elif type == "dd": type = "dword" + elif type == "dq": type = "qword" + else: raise Exception(f"Unexpected type: '{type}' (i = {i})") + # Process comment + if comment == "": comment = "Undocumented" + else: + comment = comment.lstrip() + if (len(comment) == 0): + comment = "!!! EMPTY_COMMENT" + warnings += f"{asm_file_name}:{line_idx + 1}: Empty comment in\n" + if comment[0].islower(): + warnings += f"{asm_file_name}:{line_idx + 1}: Сomment sarting with lowercase\n" + # Build the result + result = AsmVariable(line_idx + 1, name, type, init, comment, 1) + return (1, result) + +def is_id(c): + return c.isprintable() and c not in "+-/*=<>()[]{}:,|&~#`'\" \n\r\t\v" + +def get_comment_begin(line): + result = len(line) + in_str = False + for i in range(len(line)): + if in_str: + if line[i] == in_str: in_str = False + i += 1 + elif line[i] == '\'' or line[i] == '\"': + in_str = line[i] + i += 1 + elif line[i] == ';': + result = i + break + else: + i += 1 + return result + +def get_comment(line): + return line[get_comment_begin(line):] + +def remove_comment(line): + return line[0:get_comment_begin(line)] + +def insert_comment(line, comment): + comment_begin = get_comment_begin(line) + line_left = line[:get_comment_begin(line)] + line_right = line[get_comment_begin(line):] + return line_left + comment + line_right + +def has_line_wrap(line): + if remove_comment(line).rstrip()[-1] == '\\': + return True + return False + +def remove_line_wrap(line): + if remove_comment(line).rstrip()[-1] == '\\': + return remove_comment(line).rstrip()[:-1] + return line + +def parse_macro(asm_file_name, lines, line_idx): + line_idx_orig = line_idx + global warnings + + def curr(): + try: return line[i] + except: return '' + + # Returns current and then increments current index + def step(): + nonlocal i + c = curr() + i += 1 + return c + + line = lines[line_idx] + # Handle line wraps ('\' at the end) + while has_line_wrap(line): + next_line = lines[line_idx + 1] + prev_line_comment = get_comment(line) + line = remove_line_wrap(line) + insert_comment(next_line, prev_line_comment) + line_idx += 1 + + i = 0 + # Skip first spaces + while curr().isspace(): step() + # Read "macro" keyword + keyword = "" + while is_id(curr()): keyword += step() + if keyword != "macro": raise Exception(f"Not a macro: {line}") + # Skip spaces after "macro" + while curr().isspace(): step() + # Read macro name + name = "" + while curr() and not curr().isspace(): name += step() + # Skip spaces after macro name + while curr().isspace(): step() + # Find all arguments + args = [] + arg = '' + while curr() and curr() != ';' and curr() != '{': + # Collect identifier + if is_id(curr()): + arg += step() + # Save the collected identifier + elif curr() == ',': + args.append(arg) + arg = '' + step() + # Just push the '[' + elif curr() == '[': + args.append(step()) + # Just push the identifier and get ']' ready to be pushed on next comma + elif curr() == ']': + args.append(arg) + arg = step() + # Just push the identifier and get '*' ready to be pushed on next comma + elif curr() == '*': + args.append(arg) + arg = step() + # Just skip whitespaces + elif curr().isspace(): + step() + # Something unexpected + else: + raise Exception(f"Unexpected symbol '{curr()}' at index #{i} " + + f"in the macro declaration:\n'{line}'") + if arg != '': + args.append(arg) + if len(args) > 0: + print(line, args) + # Find a comment if any + comment = "" + while curr() and curr() != ';': step() + if curr() == ';': + step() + while curr(): comment += step() + # Find end of the macro + end_of_macro = False + while not end_of_macro: + line = lines[line_idx] + rbraces = re.finditer('}', line) + for rbrace_match in rbraces: + rbrace_idx = rbrace_match.start() + if line[rbrace_idx - 1] != '\\': + end_of_macro = True + line_idx += 1 + # Process comment + if comment != "": + comment = comment.lstrip() + if (len(comment) == 0): + comment = "!!! EMPTY_COMMENT" + warnings += f"{asm_file_name}:{line_idx + 1}: Empty comment in\n" + if comment[0].islower(): + warnings += f"{asm_file_name}:{line_idx + 1}: Сomment sarting with lowercase\n" + # Build the output + line_span = line_idx - line_idx_orig + 1 + result = AsmMacro(line_idx_orig, name, comment, args) + return (line_span, result) + def get_declarations(asm_file_contents, asm_file_name): asm_file_name = asm_file_name.replace("./", "") kernel_structure[asm_file_name] = [ [], [], [], [], [] ] - variable_pattern = re.compile(r'^\s*([\w\.]+)\s+d([bwdq])\s+([^;]*)\s*([;].*)?') + variable_pattern = re.compile(r'^\s*[\w\.]+\s+d[bwdq]\s+.*') macro_pattern = re.compile(r'^\s*macro\s+([\w]+).*') proc_pattern = re.compile(r'^\s*proc\s+([\w\.]+).*') label_pattern = re.compile(r'^(?!;)\s*([\w\.]+):.*') @@ -55,43 +290,23 @@ def get_declarations(asm_file_contents, asm_file_name): while line_idx < len(lines): line = lines[line_idx] - match = variable_pattern.findall(line) - if len(match) > 0: - (var_name, var_type, var_init, var_comm) = match[0] - if var_comm == "": - var_comm = "Undocumented" - else: - var_comm = var_comm[1:].lstrip() - if (len(var_comm) == 0): - var_comm = "!!! EMPTY_COMMENT" - if var_comm[0].islower(): var_comm = "!!! LOWERCASE COMMENT " + var_comm - if var_type == "b": var_type = "byte" - if var_type == "w": var_type = "word" - if var_type == "d": var_type = "dword" - if var_type == "q": var_type = "qword" - kernel_structure[asm_file_name][VARIABLES].append([ line_idx + 1, var_name, var_type, var_init, var_comm ]) - line_idx += 1 + if variable_pattern.match(line): + (skip_lines, var) = parse_variable(asm_file_name, lines, line_idx) + kernel_structure[asm_file_name][VARIABLES].append(var) + line_idx += skip_lines continue match = macro_pattern.findall(line) if len(match) > 0: - macro_name = match[0] - kernel_structure[asm_file_name][MACROS].append([ line_idx + 1, macro_name ]) - end_of_macro = False - while not end_of_macro: - line = lines[line_idx] - rbraces = re.finditer('}', line) - for rbrace_match in rbraces: - rbrace_idx = rbrace_match.start() - if line[rbrace_idx - 1] != '\\': - end_of_macro = True - line_idx += 1 + (skip_lines, macro) = parse_macro(asm_file_name, lines, line_idx) + kernel_structure[asm_file_name][MACROS].append(macro) + line_idx += skip_lines continue match = proc_pattern.findall(line) if len(match) > 0: proc_name = match[0] - kernel_structure[asm_file_name][PROCEDURES].append([ line_idx + 1, proc_name ]) + kernel_structure[asm_file_name][PROCEDURES].append(AsmFunction(line_idx + 1, proc_name)) line_idx += 1 continue @@ -100,14 +315,14 @@ def get_declarations(asm_file_contents, asm_file_name): label_name = match[0] # Don't count local labels if label_name[0] != '.': - kernel_structure[asm_file_name][LABELS].append([ line_idx + 1, label_name ]) + kernel_structure[asm_file_name][LABELS].append(AsmLabel(line_idx + 1, label_name)) line_idx += 1 continue match = struct_pattern.findall(line) if len(match) > 0: struct_name = match[0] - kernel_structure[asm_file_name][STRUCTURES].append([ line_idx + 1, struct_name ]) + kernel_structure[asm_file_name][STRUCTURES].append(AsmStruct(line_idx + 1, struct_name)) end_of_struct = False while not end_of_struct: line = lines[line_idx] @@ -149,23 +364,23 @@ if dump_symbols: if len(kernel_structure[source][VARIABLES]) > 0: print(" Variables:") for variable in kernel_structure[source][VARIABLES]: - print(f" {variable[0]}: {variable[1]}") + print(f" {variable.line}: {variable.name}") if len(kernel_structure[source][PROCEDURES]) > 0: print(" Procedures:") for procedure in kernel_structure[source][PROCEDURES]: - print(f" {procedure[0]}: {procedure[1]}") + print(f" {procedure.line}: {procedure.name}") if len(kernel_structure[source][LABELS]) > 0: print(" Global labels:") for label in kernel_structure[source][LABELS]: - print(f" {label[0]}: {label[1]}") + print(f" {label.line}: {label.name}") if len(kernel_structure[source][MACROS]) > 0: print(" Macroses:") for macro in kernel_structure[source][MACROS]: - print(f" {macro[0]}: {macro[1]}") + print(f" {macro.line}: {macro.name}") if len(kernel_structure[source][STRUCTURES]) > 0: print(" Structures:") for struct in kernel_structure[source][STRUCTURES]: - print(f" {struct[0]}: {struct[1]}") + print(f" {struct.line}: {struct.name}") if print_stats: # Collect stats @@ -208,8 +423,12 @@ def write_something(source, somehing): f.write(somehing) f.close() -def write_variable(source, line, name, type, init, brief): - name = name.replace(".", "_") +def write_variable(source, variable): + line = variable.line + type = variable.type + init = variable.init + brief = variable.comment + name = variable.name.replace(".", "_") something = (f"/**\n" + f" * @brief {brief}\n" + f" * @par Initial value\n" + @@ -240,14 +459,31 @@ def write_label(source, line, name, brief = "Undocumented"): f"void {name}();\n\n") write_something(source, something) -def write_macro(source, line, name, brief = "Undocumented"): - name = name.replace(".", "_") +def write_macro(source, macro): + if macro.comment == "": brief = "Undocumented" + else: brief = macro.comment + line = macro.line + name = macro.name.replace(".", "_").replace("@", "_") + # Construct arg list without '['s, ']'s and '*'s + args = [arg for arg in macro.args if arg not in "[]*"] + # Construct C-like arg list + arg_list = "" + if len(args) > 0: + arg_list += '(' + argc = 0 + for arg in args: + if argc != 0: + arg_list += ", " + arg_list += arg + argc += 1 + arg_list += ')' + something = (f"/**\n" + f" * @def {name}\n" + f" * @brief {brief}\n" + f" * @par Source\n" + f" * {source}:{line}\n" + - f" */\n#define {name}\n\n") + f" */\n#define {name}{arg_list}\n\n") write_something(source, something) def write_structure(source, line, name, brief = "Undocumented"): @@ -267,17 +503,20 @@ for source in kernel_structure: # Write variables doxygen of the source file if len(kernel_structure[source][VARIABLES]) > 0: for variable in kernel_structure[source][VARIABLES]: - write_variable(source, variable[0], variable[1], variable[2], variable[3], variable[4]) + write_variable(source, variable) if len(kernel_structure[source][PROCEDURES]) > 0: for procedure in kernel_structure[source][PROCEDURES]: - write_procedure(source, procedure[0], procedure[1]) + write_procedure(source, procedure.line, procedure.name) if len(kernel_structure[source][LABELS]) > 0: for label in kernel_structure[source][LABELS]: - write_label(source, label[0], label[1]) + write_label(source, label.line, label.name) if len(kernel_structure[source][MACROS]) > 0: for macro in kernel_structure[source][MACROS]: - write_macro(source, macro[0], macro[1]) + write_macro(source, macro) if len(kernel_structure[source][STRUCTURES]) > 0: for structure in kernel_structure[source][STRUCTURES]: - write_structure(source, structure[0], structure[1]) + write_structure(source, structure.line, structure.name) i += 1 + +if enable_warnings: + open('asmxygen.txt', "w", encoding = "utf-8").write(warnings)