[kernel][asmxygen.py] Implement macro arguments parsing

git-svn-id: svn://kolibrios.org@8855 a494cfbc-eb01-0410-851d-a64ba20cac60

kernel/trunk/asmxygen.py

@@ -11,21 +11,28 @@ clean_generated_stuff = False
 dump_symbols = False
 # Print symbol stats: --stats
 print_stats = False
+# Do not write warnings file: --nowarn
+enable_warnings = True
 
 # Constants
 link_root = "http://websvn.kolibrios.org/filedetails.php?repname=Kolibri+OS&path=/kernel/trunk"
 
+# Warning list
+warnings = ""
+
 # Parse arguments
 parser = argparse.ArgumentParser()
 parser.add_argument("-o", help="Doxygen output folder")
 parser.add_argument("--clean", help="Remove generated files", action="store_true")
 parser.add_argument("--dump", help="Dump all defined symbols", action="store_true")
 parser.add_argument("--stats", help="Print symbol stats", action="store_true")
+parser.add_argument("--nowarn", help="Do not write warnings file", action="store_true")
 args = parser.parse_args()
 doxygen_src_path = args.o if args.o else 'docs/doxygen'
 clean_generated_stuff = args.clean
 dump_symbols = args.dump
 print_stats = args.stats
+enable_warnings = not args.nowarn
 
 # kernel_structure["filename"] = {
 #   [ [],    # [0] Variables - [ line, name ]
@@ -40,11 +47,239 @@ LABELS = 3
 STRUCTURES = 4
 kernel_structure = {}
 
+class AsmVariable:
+	def __init__(self, line, name, type, init, comment, line_span):
+		self.line = line
+		self.name = name
+		self.type = type
+		self.init = init
+		self.comment = comment # Comment after the definition (a dd 0 ; Comment)
+		self.line_span = line_span # How much .asm lines its definition takes
+
+class AsmFunction:
+	def __init__(self, line, name):
+		self.line = line
+		self.name = name
+
+class AsmLabel:
+	def __init__(self, line, name):
+		self.line = line
+		self.name = name
+
+class AsmMacro:
+	def __init__(self, line, name, comment, args):
+		self.line = line
+		self.name = name
+		self.comment = comment
+		self.args = args
+
+class AsmStruct:
+	def __init__(self, line, name):
+		self.line = line
+		self.name = name
+
+def parse_variable(asm_file_name, lines, line_idx):
+	global warnings
+
+	def curr():
+		try: return line[i]
+		except: return ''
+
+	# Returns current and then increments current index
+	def step():
+		nonlocal i
+		c = curr()
+		i += 1
+		return c
+
+	line = lines[line_idx]
+	i = 0
+	# Skip first spaces
+	while curr().isspace(): step()
+	# Get name
+	name = ""
+	while curr().isalnum() or curr() == '_' or curr() == '.': name += step()
+	# Skip spaces after variable name
+	while curr().isspace(): step()
+	# Get type specifier (db, dd, etc.)
+	type = ""
+	while curr().isalnum() or curr() == '_': type += step()
+	# Skip spaces after type specifier
+	while curr().isspace(): step()
+	# Get initial value (everything up to end of the line or comment)
+	init = ""
+	while curr() and curr() != ';': init += step()
+	# Get comment
+	comment = ""
+	if curr() == ';':
+		step() # Skip ';'
+		while curr(): comment += step()
+	# Process type
+	if type == "db": type = "byte"
+	elif type == "dw": type = "word"
+	elif type == "dd": type = "dword"
+	elif type == "dq": type = "qword"
+	else: raise Exception(f"Unexpected type: '{type}' (i = {i})")
+	# Process comment
+	if comment == "": comment = "Undocumented"
+	else:
+		comment = comment.lstrip()
+		if (len(comment) == 0):
+			comment = "!!! EMPTY_COMMENT"
+			warnings += f"{asm_file_name}:{line_idx + 1}: Empty comment in\n"
+		if comment[0].islower():
+			warnings += f"{asm_file_name}:{line_idx + 1}: Сomment sarting with lowercase\n"
+	# Build the result
+	result = AsmVariable(line_idx + 1, name, type, init, comment, 1)
+	return (1, result)
+
+def is_id(c):
+	return c.isprintable() and c not in "+-/*=<>()[]{}:,|&~#`'\" \n\r\t\v"
+
+def get_comment_begin(line):
+	result = len(line)
+	in_str = False
+	for i in range(len(line)):
+		if in_str:
+			if line[i] == in_str: in_str = False
+			i += 1
+		elif line[i] == '\'' or line[i] == '\"':
+			in_str = line[i]
+			i += 1
+		elif line[i] == ';':
+			result = i
+			break
+		else:
+			i += 1
+	return result
+
+def get_comment(line):
+	return line[get_comment_begin(line):]
+
+def remove_comment(line):
+	return line[0:get_comment_begin(line)]
+
+def insert_comment(line, comment):
+	comment_begin = get_comment_begin(line)
+	line_left = line[:get_comment_begin(line)]
+	line_right = line[get_comment_begin(line):]
+	return line_left + comment + line_right
+
+def has_line_wrap(line):
+	if remove_comment(line).rstrip()[-1] == '\\':
+		return True
+	return False
+
+def remove_line_wrap(line):
+	if remove_comment(line).rstrip()[-1] == '\\':
+		return remove_comment(line).rstrip()[:-1]
+	return line
+
+def parse_macro(asm_file_name, lines, line_idx):
+	line_idx_orig = line_idx
+	global warnings
+
+	def curr():
+		try: return line[i]
+		except: return ''
+
+	# Returns current and then increments current index
+	def step():
+		nonlocal i
+		c = curr()
+		i += 1
+		return c
+
+	line = lines[line_idx]
+	# Handle line wraps ('\' at the end)
+	while has_line_wrap(line):
+		next_line = lines[line_idx + 1]
+		prev_line_comment = get_comment(line)
+		line = remove_line_wrap(line) + insert_comment(next_line, prev_line_comment)
+		line_idx += 1
+
+	i = 0
+	# Skip first spaces
+	while curr().isspace(): step()
+	# Read "macro" keyword
+	keyword = ""
+	while is_id(curr()): keyword += step()
+	if keyword != "macro": raise Exception(f"Not a macro: {line}")
+	# Skip spaces after "macro"
+	while curr().isspace(): step()
+	# Read macro name
+	name = ""
+	while curr() and not curr().isspace(): name += step()
+	# Skip spaces after macro name
+	while curr().isspace(): step()
+	# Find all arguments
+	args = []
+	arg = ''
+	while curr() and curr() != ';' and curr() != '{':
+		# Collect identifier
+		if is_id(curr()):
+			arg += step()
+		# Save the collected identifier
+		elif curr() == ',':
+			args.append(arg)
+			arg = ''
+			step()
+		# Just push the '['
+		elif curr() == '[':
+			args.append(step())
+		# Just push the identifier and get ']' ready to be pushed on next comma
+		elif curr() == ']':
+			args.append(arg)
+			arg = step()
+		# Just push the identifier and get '*' ready to be pushed on next comma
+		elif curr() == '*':
+			args.append(arg)
+			arg = step()
+		# Just skip whitespaces
+		elif curr().isspace():
+			step()
+		# Something unexpected
+		else:
+			raise Exception(f"Unexpected symbol '{curr()}' at index #{i} " + 
+			                f"in the macro declaration:\n'{line}'")
+	if arg != '':
+		args.append(arg)
+	if len(args) > 0:
+		print(line, args)
+	# Find a comment if any
+	comment = ""
+	while curr() and curr() != ';': step()
+	if curr() == ';':
+		step()
+		while curr(): comment += step()
+	# Find end of the macro
+	end_of_macro = False
+	while not end_of_macro:
+		line = lines[line_idx]
+		rbraces = re.finditer('}', line)
+		for rbrace_match in rbraces:
+			rbrace_idx = rbrace_match.start()
+			if line[rbrace_idx - 1] != '\\':
+				end_of_macro = True
+		line_idx += 1
+	# Process comment
+	if comment != "":
+		comment = comment.lstrip()
+		if (len(comment) == 0):
+			comment = "!!! EMPTY_COMMENT"
+			warnings += f"{asm_file_name}:{line_idx + 1}: Empty comment in\n"
+		if comment[0].islower():
+			warnings += f"{asm_file_name}:{line_idx + 1}: Сomment sarting with lowercase\n"
+	# Build the output
+	line_span = line_idx - line_idx_orig + 1
+	result = AsmMacro(line_idx_orig, name, comment, args)
+	return (line_span, result)
+
 def get_declarations(asm_file_contents, asm_file_name):
 	asm_file_name = asm_file_name.replace("./", "")
 	kernel_structure[asm_file_name] = [ [], [], [], [], [] ]
 
-	variable_pattern = re.compile(r'^\s*([\w\.]+)\s+d([bwdq])\s+([^;]*)\s*([;].*)?')
+	variable_pattern = re.compile(r'^\s*[\w\.]+\s+d[bwdq]\s+.*')
 	macro_pattern = re.compile(r'^\s*macro\s+([\w]+).*')
 	proc_pattern = re.compile(r'^\s*proc\s+([\w\.]+).*')
 	label_pattern = re.compile(r'^(?!;)\s*([\w\.]+):.*')
@@ -55,43 +290,23 @@ def get_declarations(asm_file_contents, asm_file_name):
 	while line_idx < len(lines):
 		line = lines[line_idx]
 
-		match = variable_pattern.findall(line)
+		if variable_pattern.match(line):
-		if len(match) > 0:
+			(skip_lines, var) = parse_variable(asm_file_name, lines, line_idx)
-			(var_name, var_type, var_init, var_comm) = match[0]
+			kernel_structure[asm_file_name][VARIABLES].append(var)
-			if var_comm == "":
+			line_idx += skip_lines
-				var_comm = "Undocumented"
-			else:
-				var_comm = var_comm[1:].lstrip()
-				if (len(var_comm) == 0):
-					var_comm = "!!! EMPTY_COMMENT"
-				if var_comm[0].islower(): var_comm = "!!! LOWERCASE COMMENT " + var_comm
-			if var_type == "b": var_type = "byte"
-			if var_type == "w": var_type = "word"
-			if var_type == "d": var_type = "dword"
-			if var_type == "q": var_type = "qword"
-			kernel_structure[asm_file_name][VARIABLES].append([ line_idx + 1, var_name, var_type, var_init, var_comm ])
-			line_idx += 1
 			continue
 
 		match = macro_pattern.findall(line)
 		if len(match) > 0:
-			macro_name = match[0]
+			(skip_lines, macro) = parse_macro(asm_file_name, lines, line_idx)
-			kernel_structure[asm_file_name][MACROS].append([ line_idx + 1, macro_name ])
+			kernel_structure[asm_file_name][MACROS].append(macro)
-			end_of_macro = False
+			line_idx += skip_lines
-			while not end_of_macro:
-				line = lines[line_idx]
-				rbraces = re.finditer('}', line)
-				for rbrace_match in rbraces:
-					rbrace_idx = rbrace_match.start()
-					if line[rbrace_idx - 1] != '\\':
-						end_of_macro = True
-				line_idx += 1
 			continue
 
 		match = proc_pattern.findall(line)
 		if len(match) > 0:
 			proc_name = match[0]
-			kernel_structure[asm_file_name][PROCEDURES].append([ line_idx + 1, proc_name ])
+			kernel_structure[asm_file_name][PROCEDURES].append(AsmFunction(line_idx + 1, proc_name))
 			line_idx += 1
 			continue
 
@@ -100,14 +315,14 @@ def get_declarations(asm_file_contents, asm_file_name):
 			label_name = match[0]
 			# Don't count local labels
 			if label_name[0] != '.':
-				kernel_structure[asm_file_name][LABELS].append([ line_idx + 1, label_name ])
+				kernel_structure[asm_file_name][LABELS].append(AsmLabel(line_idx + 1, label_name))
 				line_idx += 1
 				continue
 
 		match = struct_pattern.findall(line)
 		if len(match) > 0:
 			struct_name = match[0]
-			kernel_structure[asm_file_name][STRUCTURES].append([ line_idx + 1, struct_name ])
+			kernel_structure[asm_file_name][STRUCTURES].append(AsmStruct(line_idx + 1, struct_name))
 			end_of_struct = False
 			while not end_of_struct:
 				line = lines[line_idx]
@@ -149,23 +364,23 @@ if dump_symbols:
 		if len(kernel_structure[source][VARIABLES]) > 0:
 			print(" Variables:")
 			for variable in kernel_structure[source][VARIABLES]:
-				print(f"  {variable[0]}: {variable[1]}")
+				print(f"  {variable.line}: {variable.name}")
 		if len(kernel_structure[source][PROCEDURES]) > 0:
 			print(" Procedures:")
 			for procedure in kernel_structure[source][PROCEDURES]:
-				print(f"  {procedure[0]}: {procedure[1]}")
+				print(f"  {procedure.line}: {procedure.name}")
 		if len(kernel_structure[source][LABELS]) > 0:
 			print(" Global labels:")
 			for label in kernel_structure[source][LABELS]:
-				print(f"  {label[0]}: {label[1]}")
+				print(f"  {label.line}: {label.name}")
 		if len(kernel_structure[source][MACROS]) > 0:
 			print(" Macroses:")
 			for macro in kernel_structure[source][MACROS]:
-				print(f"  {macro[0]}: {macro[1]}")
+				print(f"  {macro.line}: {macro.name}")
 		if len(kernel_structure[source][STRUCTURES]) > 0:
 			print(" Structures:")
 			for struct in kernel_structure[source][STRUCTURES]:
-				print(f"  {struct[0]}: {struct[1]}")
+				print(f"  {struct.line}: {struct.name}")
 
 if print_stats:
 	# Collect stats
@@ -208,8 +423,12 @@ def write_something(source, somehing):
 	f.write(somehing)
 	f.close()
 
-def write_variable(source, line, name, type, init, brief):
+def write_variable(source, variable):
-	name = name.replace(".", "_")
+	line = variable.line
+	type = variable.type
+	init = variable.init
+	brief = variable.comment
+	name = variable.name.replace(".", "_")
 	something = (f"/**\n" +
 	             f" * @brief {brief}\n" +
 	             f" * @par Initial value\n" +
@@ -240,14 +459,31 @@ def write_label(source, line, name, brief = "Undocumented"):
 	             f"void {name}();\n\n")
 	write_something(source, something)
 
-def write_macro(source, line, name, brief = "Undocumented"):
+def write_macro(source, macro):
-	name = name.replace(".", "_")
+	if macro.comment == "": brief = "Undocumented"
+	else: brief = macro.comment
+	line = macro.line
+	name = macro.name.replace(".", "_").replace("@", "_")
+	# Construct arg list without '['s, ']'s and '*'s
+	args = [arg for arg in macro.args if arg not in "[]*"]
+	# Construct C-like arg list
+	arg_list = ""
+	if len(args) > 0:
+		arg_list += '('
+		argc = 0
+		for arg in args:
+			if argc != 0:
+				arg_list += ", "
+			arg_list += arg
+			argc += 1
+		arg_list += ')'
+
 	something = (f"/**\n" +
 	             f" * @def {name}\n" +
 	             f" * @brief {brief}\n" +
 	             f" * @par Source\n" +
 	             f" * <a href='{link_root}/{source}#line-{line}'>{source}:{line}</a>\n" +
-	             f" */\n#define {name}\n\n")
+	             f" */\n#define {name}{arg_list}\n\n")
 	write_something(source, something)
 
 def write_structure(source, line, name, brief = "Undocumented"):
@@ -267,17 +503,20 @@ for source in kernel_structure:
 	# Write variables doxygen of the source file
 	if len(kernel_structure[source][VARIABLES]) > 0:
 		for variable in kernel_structure[source][VARIABLES]:
-			write_variable(source, variable[0], variable[1], variable[2], variable[3], variable[4])
+			write_variable(source, variable)
 	if len(kernel_structure[source][PROCEDURES]) > 0:
 		for procedure in kernel_structure[source][PROCEDURES]:
-			write_procedure(source, procedure[0], procedure[1])
+			write_procedure(source, procedure.line, procedure.name)
 	if len(kernel_structure[source][LABELS]) > 0:
 		for label in kernel_structure[source][LABELS]:
-			write_label(source, label[0], label[1])
+			write_label(source, label.line, label.name)
 	if len(kernel_structure[source][MACROS]) > 0:
 		for macro in kernel_structure[source][MACROS]:
-			write_macro(source, macro[0], macro[1])
+			write_macro(source, macro)
 	if len(kernel_structure[source][STRUCTURES]) > 0:
 		for structure in kernel_structure[source][STRUCTURES]:
-			write_structure(source, structure[0], structure[1])
+			write_structure(source, structure.line, structure.name)
 	i += 1
+
+if enable_warnings:
+	open('asmxygen.txt', "w", encoding = "utf-8").write(warnings)