XenOn/kolibrios-fun
1
1
Fork 0
forked from KolibriOS/kolibrios
Code Pull Requests Wiki Activity

[ASMXYGEN] Conform PEP8 except line import re

Browse Source 
git-svn-id: svn://kolibrios.org@9408 a494cfbc-eb01-0410-851d-a64ba20cac60
This commit is contained in:
Magomed Kostoev (mkostoevr) 2021-12-07 19:35:46 +00:00
parent c867c5020b
commit 6aea287961
1 changed files with 278 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

188
kernel/trunk/asmxygen.py
View File

@ -166,18 +166,21 @@ fasm_types = [
"du",
]
# Add kind flag to identifier in id2kind
def id_add_kind(identifier, kind):
if identifier not in id2kind:
id2kind[identifier] = ''
id2kind[identifier] += kind
# Remove kind flag of identifier in id2kind
def id_remove_kind(identifier, kind):
if identifier in id2kind:
if kind in id2kind[identifier]:
id2kind[identifier] = id2kind[identifier].replace(kind, '')
# Get kind of an identifier
def id_get_kind(identifier):
if identifier in id2kind:
@ -185,6 +188,7 @@ def id_get_kind(identifier):
else:
return ''
class LegacyAsmReader:
def __init__(self, file):
self.file = file
@ -196,8 +200,10 @@ class LegacyAsmReader:
return self.lines[self.line_idx]
def curr(self):
try: return self.lines[self.line_idx][self.i]
except: return ''
try:
return self.lines[self.line_idx][self.i]
except:
return ''
def step(self):
c = self.curr()
@ -237,6 +243,7 @@ class LegacyAsmReader:
while self.curr().isspace():
self.step()
class AsmReaderRecognizingStrings(LegacyAsmReader):
def __init__(self, file):
super().__init__(file)
@ -247,9 +254,9 @@ class AsmReaderRecognizingStrings(LegacyAsmReader):
c = super().step()
if self.should_recognize_strings and (c == '"' or c == "'"):
# If just now we was at the double or single quotation mark
# and we aren't in a string yet
# then say "we are in a string openned with this quotation mark now"
if self.in_string == None:
# and we aren't in a string yet then say
# "we are in a string openned with this quotation mark now"
if self.in_string is None:
self.in_string = c
# If just now we was at the double or single quotation mark
# and we are in the string entered with the same quotation mark
@ -258,6 +265,7 @@ class AsmReaderRecognizingStrings(LegacyAsmReader):
self.in_string = None
return c
class AsmReaderReadingComments(AsmReaderRecognizingStrings):
def __init__(self, file):
super().__init__(file)
@ -282,8 +290,11 @@ class AsmReaderReadingComments(AsmReaderRecognizingStrings):
self.status_has_code = True
def update_status(self):
# If we aren't in a comment and we aren't in a string - say we are now in a comment if ';' met
if not self.status_has_comment and not self.in_string and self.curr() == ';':
# If we aren't in a comment and we aren't in a string -
# say we are now in a comment if ';' met
if (not self.status_has_comment and
not self.in_string and
self.curr() == ';'):
self.status_set_has_comment()
# Else if we are in a comment - collect the comment
elif self.status_has_comment:
@ -305,7 +316,8 @@ class AsmReaderReadingComments(AsmReaderRecognizingStrings):
super().nextline()
# If the line we leave was not a comment-only line
# then forget the collected comment
# Otherwise the collected comment should be complemented by comment from next line in step()
# Otherwise the collected comment should be complemented by
# comment from next line in step()
if self.status_has_code:
# But we should preserve comment for the next line
# If previous line set align (cause many functions re documented
@ -314,9 +326,11 @@ class AsmReaderReadingComments(AsmReaderRecognizingStrings):
self.comment = ''
# Reset the line status (now it's the status of the new line)
self.status_reset()
# Set new status for this line according to the first character in the line
# Set new status for this line according to the
# first character in the line
self.update_status()
class AsmReaderFetchingIdentifiers(AsmReaderReadingComments):
def __init__(self, file):
super().__init__(file)
@ -328,10 +342,12 @@ class AsmReaderFetchingIdentifiers(AsmReaderReadingComments):
result += self.step()
return result
class AsmReader(AsmReaderFetchingIdentifiers):
def __init__(self, file):
super().__init__(file)
def append_file(full_path, contents):
if debug_mode:
if full_path not in output_files:
@ -342,11 +358,13 @@ def append_file(full_path, contents):
f.write(contents)
f.close()
class AsmElement:
def __init__(self, location, name, comment):
global warnings
# If the element was constructed during this execution then the element is new
# If the element was constructed during this execution then
# the element is new
self.new = True
self.location = location
self.file = self.location.split(':')[0].replace('\\', '/')
@ -361,7 +379,7 @@ class AsmElement:
print(f"\n{self.location}: {self.name}")
print(f"{self.comment}")
def emit(self, dest, doxycomment = '', declaration = ''):
def emit(self, dest, doxycomment='', declaration=''):
# Do not emit anything if the symbol is marked as hidden in its comment
if '@dont_give_a_doxygen' in self.comment:
return
@ -374,19 +392,22 @@ class AsmElement:
if not doxycomment.endswith('\n'):
doxycomment += '\n'
if doxycomment.split('@brief ')[1][0].islower():
warnings += f"{self.location}: Brief comment starting from lowercase\n"
warnings += (f"{self.location}: Brief comment starting from " +
"lowercase\n")
# Build contents to emit
contents = ''
contents += '/**\n'
contents += doxycomment
contents += (f"@par Source\n" +
f"<a href='{link_root}/{self.file}#line-{self.line}'>{self.file}:{self.line}</a>\n")
f"<a href='{link_root}/{self.file}" +
f"#line-{self.line}'>{self.file}:{self.line}</a>\n")
contents += '*/\n'
contents += declaration
contents += '\n\n'
# Get path to file to emit this
full_path = dest + '/' + self.file
# Remove the file on first access if it was created by previous generation
# Remove the file on first access if it was
# created by previous generation
if full_path not in created_files:
if os.path.isfile(full_path):
os.remove(full_path)
@ -397,6 +418,7 @@ class AsmElement:
append_file(full_path, contents)
class AsmVariable(AsmElement):
def __init__(self, location, name, comment, type, init):
super().__init__(location, name, comment)
@ -422,8 +444,10 @@ class AsmVariable(AsmElement):
# Emit this
super().emit(dest, doxycomment, declaration)
class AsmFunction(AsmElement):
def __init__(self, location, name, comment, calling_convention, args, used_regs):
def __init__(self, location, name, comment, calling_convention,
args, used_regs):
super().__init__(location, name, comment)
self.calling_convention = calling_convention
self.args = args
@ -473,6 +497,7 @@ class AsmFunction(AsmElement):
# Emit this
super().emit(dest, doxycomment, declaration)
class AsmLabel(AsmElement):
def __init__(self, location, name, comment):
super().__init__(location, name, comment)
@ -493,6 +518,7 @@ class AsmLabel(AsmElement):
# Emit this
super().emit(dest, doxycomment, declaration)
class AsmMacro(AsmElement):
def __init__(self, location, name, comment, args):
super().__init__(location, name, comment)
@ -526,6 +552,7 @@ class AsmMacro(AsmElement):
# Emit this
super().emit(dest, doxycomment, declaration)
class AsmStruct(AsmElement):
def __init__(self, location, name, comment, members):
super().__init__(location, name, comment)
@ -546,11 +573,13 @@ class AsmStruct(AsmElement):
declaration = f"struct {self.name}" + " {\n"
for member in self.members:
if type(member) == AsmVariable:
declaration += f'\t{member.type} {member.name}; /**< {member.comment} */\n'
declaration += (f'\t{member.type} {member.name}; ' +
f'/**< {member.comment} */\n')
declaration += '};'
# Emit this
super().emit(dest, doxycomment, declaration)
class AsmUnion(AsmElement):
def __init__(self, location, name, comment, members):
super().__init__(location, name, comment)
@ -571,16 +600,20 @@ class AsmUnion(AsmElement):
# Emit this
super().emit(dest, doxycomment, declaration)
class VariableNameIsMacroName:
def __init__(self, name):
self.name = name
def is_id(c):
return c.isprintable() and c not in "+-/*=<>()[]{};:,|&~#`'\" \n\r\t\v"
def is_starts_as_id(s):
return not s[0].isdigit()
def parse_after_macro(r):
location = r.location()
@ -620,15 +653,17 @@ def parse_after_macro(r):
r.step()
# Something unexpected
else:
raise Exception(f"Unexpected symbol '{r.curr()}' at index #{r.i} " +
f"in the macro declaration at {location} " +
raise Exception(f"Unexpected symbol '{r.curr()}' " +
f"at index #{r.i} in the macro declaration " +
f"at {location} " +
f"(line: {r.lines[r.line_idx]})\n''")
# Append the last argument
if arg != '':
args.append(arg)
# Skip t spaces after the argument list
r.skip_spaces()
# Get a comment if it is: read till the end of the line and get the comment from the reader
# Get a comment if it is: read till the end of the line and
# get the comment from the reader
while r.curr() != '':
r.step()
comment = r.comment
@ -645,7 +680,8 @@ def parse_after_macro(r):
# Build the output
return AsmMacro(location, name, comment, args)
def parse_variable(r, first_word = None):
def parse_variable(r, first_word=None):
global warnings
location = r.location()
@ -654,14 +690,15 @@ def parse_variable(r, first_word = None):
# Get variable name
name = ""
# Read it if it was not supplied
if first_word == None:
if first_word is None:
while is_id(r.curr()):
name += r.step()
# Or use the supplied one instead
else:
name = first_word
# Check the name
# If it's 0 len, that means threr's something else than an identifier at the beginning
# If it's 0 len, that means threr's something else than an
# identifier at the beginning
if len(name) == 0:
return None
# If it starts from digit or othervice illegally it's illegal
@ -675,7 +712,8 @@ def parse_variable(r, first_word = None):
# If it's a macro name, that's not a variable declaration
if ID_KIND_MACRO_NAME in kind:
return VariableNameIsMacroName(name)
# If it's a datatype or a structure name that's not a variable declaration: that's just a data
# If it's a datatype or a structure name that's not a
# variable declaration: that's just a data
# don't document just a data for now
if ID_KIND_STRUCT_NAME in kind or ID_KIND_FASM_TYPE in kind:
return None
@ -714,7 +752,8 @@ def parse_variable(r, first_word = None):
# Build the result
return AsmVariable(location, name, r.comment, var_type, value)
def parse_after_struct(r, as_union = True):
def parse_after_struct(r, as_union=True):
global warnings
location = r.location()
@ -740,14 +779,15 @@ def parse_after_struct(r, as_union = True):
elif type(var) == str:
if var == 'union':
# Parse the union as a struct
union = parse_after_struct(r, as_union = True)
union = parse_after_struct(r, as_union=True)
members.append(union)
# Skip the ends of the union
r.nextline()
elif r.curr() == ':':
warnings += f"{r.location()}: Skept the label in the struct\n"
else:
raise Exception(f"Garbage in struct member at {location} (got '{var}' identifier)")
raise Exception(f"Garbage in struct member at {location} " +
f" (got '{var}' identifier)")
elif type(var) == VariableNameIsMacroName:
if var.name == 'ends':
break
@ -758,6 +798,7 @@ def parse_after_struct(r, as_union = True):
else:
return AsmUnion(location, name, comment, members)
def parse_after_proc(r):
# Get proc name
name = r.fetch_identifier()
@ -813,7 +854,9 @@ def parse_after_proc(r):
r.step()
comment = r.comment
# Build the element
return AsmFunction(r.location(), name, comment, calling_convention, args, used_regs)
return AsmFunction(r.location(), name, comment, calling_convention,
args, used_regs)
def get_declarations(asm_file_contents, asm_file_name):
r = AsmReader(asm_file_name)
@ -857,7 +900,8 @@ def get_declarations(asm_file_contents, asm_file_name):
pass
elif first_word == 'purge':
while True:
# Skip spaces after the 'purge' keyword or after the comma what separated the previous macro name
# Skip spaces after the 'purge' keyword or after
# the comma what separated the previous macro name
r.skip_spaces()
# Get the purged macro name
name = ''
@ -870,7 +914,8 @@ def get_declarations(asm_file_contents, asm_file_name):
pass
# Skip spaces after the name
r.skip_spaces()
# If it's comma (',') after then that's not the last purged macro, continue purging
# If it's comma (',') after then that's not the last purged
# macro, continue purging
if r.curr() == ',':
r.step()
continue
@ -893,14 +938,19 @@ def get_declarations(asm_file_contents, asm_file_name):
name = var
# Match label beginning (':' after name)
if r.curr() == ':':
# Get to the end of the line and get the coment from the reader
# Get to the end of the line and
# get the coment from the reader
while r.curr() != '':
r.step()
comment = r.comment
# Only handle non-local labels
if name[0] != '.' and name != "@@" and name != "$Revision":
# Treate the label as function if there's @return or
# @param in its comment. Othervice it's just a variable
# with type `label` in generated doxygen C
if '@return' in comment or '@param' in comment:
element = AsmFunction(r.location(), name, comment, '', [], [])
element = AsmFunction(r.location(), name, comment,
'', [], [])
else:
element = AsmLabel(r.location(), name, comment)
elements.append(element)
@ -914,6 +964,7 @@ def get_declarations(asm_file_contents, asm_file_name):
id_add_kind(word_one, ID_KIND_EQUATED_CONSTANT)
r.nextline()
def it_neds_to_be_parsed(source_file):
# If there's no symbols file saved - parse it anyway
# cause we need to create the symbols file and use it
@ -933,7 +984,8 @@ def it_neds_to_be_parsed(source_file):
return True
return False
def handle_file(handled_files, asm_file_name, subdir = "."):
def handle_file(handled_files, asm_file_name, subdir="."):
global elements
# Canonicalize the file path and get it relative to cwd
cwd = os.path.abspath(os.path.dirname(sys.argv[0]))
@ -947,7 +999,8 @@ def handle_file(handled_files, asm_file_name, subdir = "."):
return
# Say that the file was handled in this execution
handled_files.append(asm_file_name)
# Check if the file should be parsed (if it was modified or wasn't parsed yet)
# Check if the file should be parsed
# (if it was modified or wasn't parsed yet)
should_get_declarations = True
if not it_neds_to_be_parsed(asm_file_name):
print(f"Skipping {asm_file_name} (already newest)")
@ -955,8 +1008,12 @@ def handle_file(handled_files, asm_file_name, subdir = "."):
else:
print(f"Handling {asm_file_name}")
# Remove elements parsed from this file before if any
elements_to_remove = [x for x in elements if x.location.split(':')[0] == asm_file_name]
elements = [x for x in elements if x.location.split(':')[0] != asm_file_name]
elements_to_remove = [
x for x in elements if x.location.split(':')[0] == asm_file_name
]
elements = [
x for x in elements if x.location.split(':')[0] != asm_file_name
]
# Forget types of identifiers of names of the removed elements
for element in elements_to_remove:
if type(element) == AsmStruct:
@ -966,10 +1023,11 @@ def handle_file(handled_files, asm_file_name, subdir = "."):
# Read the source
asm_file_contents = open(asm_file_name, "r", encoding="utf-8").read()
# Find includes, fix their paths and handle em recoursively
includes = re.findall(r'^include (["\'])(.*)\1', asm_file_contents, flags=re.MULTILINE)
includes = re.findall(r'^include (["\'])(.*)\1', asm_file_contents,
flags=re.MULTILINE)
for include in includes:
include = include[1].replace('\\', '/');
full_path = subdir + '/' + include;
include = include[1].replace('\\', '/')
full_path = subdir + '/' + include
# If the path isn't valid, maybe that's not relative path
if not os.path.isfile(full_path):
full_path = include
@ -980,7 +1038,8 @@ def handle_file(handled_files, asm_file_name, subdir = "."):
get_declarations(asm_file_contents, asm_file_name)
if __name__ == "__main__":
link_root = "http://websvn.kolibrios.org/filedetails.php?repname=Kolibri+OS&path=/kernel/trunk"
link_root = "http://websvn.kolibrios.org/filedetails.php"
link_root += "?repname=Kolibri+OS&path=/kernel/trunk"
# Dict where an identifier is assicoated with a string
# The string contains characters specifying flags
@ -1023,12 +1082,24 @@ if __name__ == "__main__":
# Parse arguments
parser = argparse.ArgumentParser()
parser.add_argument("-o", help="Doxygen output folder")
parser.add_argument("--clean", help="Remove generated files", action="store_true")
parser.add_argument("--dump", help="Dump all defined symbols", action="store_true")
parser.add_argument("--stats", help="Print symbol stats", action="store_true")
parser.add_argument("--nowarn", help="Do not write warnings file", action="store_true")
parser.add_argument("--noemit", help="Do not emit doxygen files (for testing)", action="store_true")
parser.add_argument("--debug", help="Show hashes of files (for testing)", action="store_true")
parser.add_argument("--clean",
help="Remove generated files",
action="store_true")
parser.add_argument("--dump",
help="Dump all defined symbols",
action="store_true")
parser.add_argument("--stats",
help="Print symbol stats",
action="store_true")
parser.add_argument("--nowarn",
help="Do not write warnings file",
action="store_true")
parser.add_argument("--noemit",
help="Do not emit doxygen files (for testing)",
action="store_true")
parser.add_argument("--debug",
help="Show hashes of files (for testing)",
action="store_true")
args = parser.parse_args()
doxygen_src_path = args.o if args.o else 'docs/doxygen'
clean_generated_stuff = args.clean
@ -1042,18 +1113,20 @@ if __name__ == "__main__":
elements = []
created_files = []
kernel_files = []
output_files = {} # If --debug then all the files are written here
output_files = {} # If --debug then all the files are written here
# Load remembered list of symbols
if os.path.isfile('asmxygen.elements.pickle'):
print('Reading existing dump of symbols')
(elements, id2kind) = pickle.load(open('asmxygen.elements.pickle', 'rb'))
pickle_file = open('asmxygen.elements.pickle', 'rb')
(elements, id2kind) = pickle.load(pickle_file)
pickle_file.close()
handle_file(kernel_files, "./kernel.asm");
handle_file(kernel_files, "./kernel.asm")
if dump_symbols:
stdout = sys.stdout
sys.stdout = open('asmxygen.dump.txt', 'w', encoding = 'utf-8')
sys.stdout = open('asmxygen.dump.txt', 'w', encoding='utf-8')
for asm_element in elements:
asm_element.dump()
sys.stdout = stdout
@ -1063,7 +1136,7 @@ if __name__ == "__main__":
for file in kernel_files:
doxygen_file = f"{doxygen_src_path}/{file}"
if (os.path.isfile(doxygen_file)):
print(f"Removing {file}... ", end = '')
print(f"Removing {file}... ", end='')
os.remove(doxygen_file)
print("Done.")
elif not noemit:
@ -1072,16 +1145,20 @@ if __name__ == "__main__":
i = 0
new_elements = [x for x in elements if x.new]
for element in new_elements:
print(f"[{i + 1}/{len(new_elements)}] Emitting {element.name} from {element.location}")
counter = f"[{i + 1}/{len(new_elements)}]"
print(f"{counter} Emitting {element.name} from {element.location}")
element.emit(doxygen_src_path)
i += 1
print(f"Writing dump of symbols to asmxygen.elements.pickle")
# Now when the new elements already was written, there's no new elements anymore
# Now when the new elements already was written, there's no new
# elements anymore
for element in elements:
element.new = False
pickle.dump((elements, id2kind), open('asmxygen.elements.pickle', 'wb'))
pickle_file = open('asmxygen.elements.pickle', 'wb')
pickle.dump((elements, id2kind), pickle_file)
pickle_file.close()
if print_stats:
var_count = 0
@ -1111,7 +1188,7 @@ if __name__ == "__main__":
print(f'Parsed structure type count: {str_count}')
if enable_warnings:
open('asmxygen.txt', "w", encoding = "utf-8").write(warnings)
open('asmxygen.txt', "w", encoding="utf-8").write(warnings)
if debug_mode:
hash_per_file = ""
@ -1124,6 +1201,7 @@ if __name__ == "__main__":
else:
reference_hash_per_file = open("asmxygen_hash_per_file.txt").read()
if reference_hash_per_file != hash_per_file:
print(''.join(difflib.ndiff(reference_hash_per_file, hash_per_file)))
diffs = difflib.ndiff(reference_hash_per_file, hash_per_file)
print(''.join(diffs))
else:
print("SUCCESS")