import re import os import argparse import sys import pickle # Parameters # Path to doxygen folder to make doxygen files in: -o doxygen_src_path = 'docs/doxygen' # Remove generated doxygen files: --clean clean_generated_stuff = False # Dump all defined symbols: --dump dump_symbols = False # Print symbol stats: --stats print_stats = False # Do not write warnings file: --nowarn enable_warnings = True # Constants link_root = "http://websvn.kolibrios.org/filedetails.php?repname=Kolibri+OS&path=/kernel/trunk" # fasm keywords keywords = [ # Generic keywords "align", "equ", "org", "while", "load", "store", "times", "repeat", "virtual", "display", "err", "assert", "if", # Instructions "aaa", "aad", "aam", "aas", "adc", "adcx", "add", "addpd", "addps", "addsd", "addss", "addsubpd", "addsubps", "adox", "aesdec", "aesdeclast", "aesenc", "aesenclast", "aesimc", "aeskeygenassist", "and", "andn", "andnpd", "andnps", "andpd", "andps", "arpl", "bextr", "blendpd", "blendps", "blendvpd", "blendvps", "blsi", "blsmsk", "blsr", "bndcl", "bndcn", "bndcu", "bndldx", "bndmk", "bndmov", "bndstx", "bound", "bsf", "bsr", "bswap", "bt", "btc", "btr", "bts", "bzhi", "call", "cbw", "cdq", "cdqe", "clac", "clc", "cld", "cldemote", "clflush", "clflushopt", "cli", "clts", "clwb", "cmc", "cmova", "cmovae", "cmovb", "cmovbe", "cmovc", "cmove", "cmovg", "cmovge", "cmovl", "cmovle", "cmovna", "cmovnae", "cmovnb", "cmovnbe", "cmovnc", "cmovne", "cmovng", "cmovnge", "cmovnl", "cmovnle", "cmovno", "cmovnp", "cmovns", "cmovnz", "cmovo", "cmovp", "cmovpe", "cmovpo", "cmovs", "cmovz", "cmp", "cmppd", "cmpps", "cmps", "cmpsb", "cmpsd", "cmpsd", "cmpsq", "cmpss", "cmpsw", "cmpxchg", "cmpxchg16b", "cmpxchg8b", "comisd", "comiss", "cpuid", "cqo", "crc32", "cvtdq2pd", "cvtdq2ps", "cvtpd2dq", "cvtpd2pi", "cvtpd2ps", "cvtpi2pd", "cvtpi2ps", "cvtps2dq", "cvtps2pd", "cvtps2pi", "cvtsd2si", "cvtsd2ss", "cvtsi2sd", "cvtsi2ss", "cvtss2sd", "cvtss2si", "cvttpd2dq", "cvttpd2pi", "cvttps2dq", "cvttps2pi", "cvttsd2si", "cvttss2si", "cwd", "cwde", "daa", "das", "dec", "div", "divpd", "divps", "divsd", "divss", "dppd", "dpps", "emms", "enter", "extractps", "f2xm1", "fabs", "fadd", "faddp", "fbld", "fbstp", "fchs", "fclex", "fcmova", "fcmovae", "fcmovb", "fcmovbe", "fcmovc", "fcmove", "fcmovg", "fcmovge", "fcmovl", "fcmovle", "fcmovna", "fcmovnae", "fcmovnb", "fcmovnbe", "fcmovnc", "fcmovne", "fcmovng", "fcmovnge", "fcmovnl", "fcmovnle", "fcmovno", "fcmovnp", "fcmovns", "fcmovnz", "fcmovo", "fcmovp", "fcmovpe", "fcmovpo", "fcmovs", "fcmovz", "fcom", "fcomi", "fcomip", "fcomp", "fcompp", "fcos", "fdecstp", "fdiv", "fdivp", "fdivr", "fdivrp", "ffree", "fiadd", "ficom", "ficomp", "fidiv", "fidivr", "fild", "fimul", "fincstp", "finit", "fist", "fistp", "fisttp", "fisub", "fisubr", "fld", "fld1", "fldcw", "fldenv", "fldl2e", "fldl2t", "fldlg2", "fldln2", "fldpi", "fldz", "fmul", "fmulp", "fnclex", "fninit", "fnop", "fnsave", "fnstcw", "fnstenv", "fnstsw", "fpatan", "fprem", "fprem1", "fptan", "frndint", "frstor", "fsave", "fscale", "fsin", "fsincos", "fsqrt", "fst", "fstcw", "fstenv", "fstp", "fstsw", "fsub", "fsubp", "fsubr", "fsubrp", "ftst", "fucom", "fucomi", "fucomip", "fucomp", "fucompp", "fwait", "fxam", "fxch", "fxrstor", "fxsave", "fxtract", "fyl2x", "fyl2xp1", "gf2p8affineinvqb", "gf2p8affineqb", "gf2p8mulb", "haddpd", "haddps", "hlt", "hsubpd", "hsubps", "idiv", "imul", "in", "inc", "ins", "insb", "insd", "insertps", "insw", "int", "int1", "int3", "into", "invd", "invlpg", "invpcid", "iret", "iretd", "jmp", "ja", "jae", "jb", "jbe", "jc", "jcxz", "jecxz", "je", "jg", "jge", "jl", "jle", "jna", "jnae", "jnb", "jnbe", "jnc", "jne", "jng", "jnge", "jnl", "jnle", "jno", "jnp", "jns", "jnz", "jo", "jp", "jpe", "jpo", "js", "jz", "kaddb", "kaddd", "kaddq", "kaddw", "kandb", "kandd", "kandnb", "kandnd", "kandnq", "kandnw", "kandq", "kandw", "kmovb", "kmovd", "kmovq", "kmovw", "knotb", "knotd", "knotq", "knotw", "korb", "kord", "korq", "kortestb", "kortestd", "kortestq", "kortestw", "korw", "kshiftlb", "kshiftld", "kshiftlq", "kshiftlw", "kshiftrb", "kshiftrd", "kshiftrq", "kshiftrw", "ktestb", "ktestd", "ktestq", "ktestw", "kunpckbw", "kunpckdq", "kunpckwd", "kxnorb", "kxnord", "kxnorq", "kxnorw", "kxorb", "kxord", "kxorq", "kxorw", "lahf", "lar", "lddqu", "ldmxcsr", "lds", "lea", "leave", "les", "lfence", "lfs", "lgdt", "lgs", "lidt", "lldt", "lmsw", "lock", "lods", "lodsb", "lodsd", "lodsq", "lodsw", "loop", "loopa", "loopae", "loopb", "loopbe", "loopc", "loope", "loopg", "loopge", "loopl", "loople", "loopna", "loopnae", "loopnb", "loopnbe", "loopnc", "loopne", "loopng", "loopnge", "loopnl", "loopnle", "loopno", "loopnp", "loopns", "loopnz", "loopo", "loopp", "looppe", "looppo", "loops", "loopz", "lsl", "lss", "ltr", "lzcnt", "maskmovdqu", "maskmovq", "maxpd", "maxps", "maxsd", "maxss", "mfence", "minpd", "minps", "minsd", "minss", "monitor", "mov", "movapd", "movaps", "movbe", "movd", "movddup", "movdir64b", "movdiri", "movdq2q", "movdqa", "movdqu", "movhlps", "movhpd", "movhps", "movlhps", "movlpd", "movlps", "movmskpd", "movmskps", "movntdq", "movntdqa", "movnti", "movntpd", "movntps", "movntq", "movq", "movq", "movq2dq", "movs", "movsb", "movsd", "movsd", "movshdup", "movsldup", "movsq", "movss", "movsw", "movsx", "movsxd", "movupd", "movups", "movzx", "mpsadbw", "mul", "mulpd", "mulps", "mulsd", "mulss", "mulx", "mwait", "neg", "nop", "not", "or", "orpd", "orps", "out", "outs", "outsb", "outsd", "outsw", "pabsb", "pabsd", "pabsq", "pabsw", "packssdw", "packsswb", "packusdw", "packuswb", "paddb", "paddd", "paddq", "paddsb", "paddsw", "paddusb", "paddusw", "paddw", "palignr", "pand", "pandn", "pause", "pavgb", "pavgw", "pblendvb", "pblendw", "pclmulqdq", "pcmpeqb", "pcmpeqd", "pcmpeqq", "pcmpeqw", "pcmpestri", "pcmpestrm", "pcmpgtb", "pcmpgtd", "pcmpgtq", "pcmpgtw", "pcmpistri", "pcmpistrm", "pdep", "pext", "pextrb", "pextrd", "pextrq", "pextrw", "phaddd", "phaddsw", "phaddw", "phminposuw", "phsubd", "phsubsw", "phsubw", "pinsrb", "pinsrd", "pinsrq", "pinsrw", "pmaddubsw", "pmaddwd", "pmaxsb", "pmaxsd", "pmaxsq", "pmaxsw", "pmaxub", "pmaxud", "pmaxuq", "pmaxuw", "pminsb", "pminsd", "pminsq", "pminsw", "pminub", "pminud", "pminuq", "pminuw", "pmovmskb", "pmovsx", "pmovzx", "pmuldq", "pmulhrsw", "pmulhuw", "pmulhw", "pmulld", "pmullq", "pmullw", "pmuludq", "pop", "popa", "popad", "popcnt", "popf", "popfd", "popfq", "por", "prefetchw", "prefetchh", "psadbw", "pshufb", "pshufd", "pshufhw", "pshuflw", "pshufw", "psignb", "psignd", "psignw", "pslld", "pslldq", "psllq", "psllw", "psrad", "psraq", "psraw", "psrld", "psrldq", "psrlq", "psrlw", "psubb", "psubd", "psubq", "psubsb", "psubsw", "psubusb", "psubusw", "psubw", "ptest", "ptwrite", "punpckhbw", "punpckhdq", "punpckhqdq", "punpckhwd", "punpcklbw", "punpckldq", "punpcklqdq", "punpcklwd", "push", "pushw", "pushd", "pusha", "pushad", "pushf", "pushfd", "pushfq", "pxor", "rcl", "rcpps", "rcpss", "rcr", "rdfsbase", "rdgsbase", "rdmsr", "rdpid", "rdpkru", "rdpmc", "rdrand", "rdseed", "rdtsc", "rdtscp", "rep", "repe", "repne", "repnz", "repz", "ret", "rol", "ror", "rorx", "roundpd", "roundps", "roundsd", "roundss", "rsm", "rsqrtps", "rsqrtss", "sahf", "sal", "sar", "sarx", "sbb", "scas", "scasb", "scasd", "scasw", "seta", "setae", "setb", "setbe", "setc", "sete", "setg", "setge", "setl", "setle", "setna", "setnae", "setnb", "setnbe", "setnc", "setne", "setng", "setnge", "setnl", "setnle", "setno", "setnp", "setns", "setnz", "seto", "setp", "setpe", "setpo", "sets", "setz", "sfence", "sgdt", "sha1msg1", "sha1msg2", "sha1nexte", "sha1rnds4", "sha256msg1", "sha256msg2", "sha256rnds2", "shl", "shld", "shlx", "shr", "shrd", "shrx", "shufpd", "shufps", "sidt", "sldt", "smsw", "sqrtpd", "sqrtps", "sqrtsd", "sqrtss", "stac", "stc", "std", "sti", "stmxcsr", "stos", "stosb", "stosd", "stosq", "stosw", "str", "sub", "subpd", "subps", "subsd", "subss", "swapgs", "syscall", "sysenter", "sysexit", "sysret", "test", "tpause", "tzcnt", "ucomisd", "ucomiss", "ud", "umonitor", "umwait", "unpckhpd", "unpckhps", "unpcklpd", "unpcklps", "valignd", "valignq", "vblendmpd", "vblendmps", "vbroadcast", "vcompresspd", "vcompressps", "vcvtpd2qq", "vcvtpd2udq", "vcvtpd2uqq", "vcvtph2ps", "vcvtps2ph", "vcvtps2qq", "vcvtps2udq", "vcvtps2uqq", "vcvtqq2pd", "vcvtqq2ps", "vcvtsd2usi", "vcvtss2usi", "vcvttpd2qq", "vcvttpd2udq", "vcvttpd2uqq", "vcvttps2qq", "vcvttps2udq", "vcvttps2uqq", "vcvttsd2usi", "vcvttss2usi", "vcvtudq2pd", "vcvtudq2ps", "vcvtuqq2pd", "vcvtuqq2ps", "vcvtusi2sd", "vcvtusi2ss", "vdbpsadbw", "verr", "verw", "vexpandpd", "vexpandps", "vextractf128", "vextractf32x4", "vextractf32x8", "vextractf64x2", "vextractf64x4", "vextracti128", "vextracti32x4", "vextracti32x8", "vextracti64x2", "vextracti64x4", "vfixupimmpd", "vfixupimmps", "vfixupimmsd", "vfixupimmss", "vfmadd132pd", "vfmadd132ps", "vfmadd132sd", "vfmadd132ss", "vfmadd213pd", "vfmadd213ps", "vfmadd213sd", "vfmadd213ss", "vfmadd231pd", "vfmadd231ps", "vfmadd231sd", "vfmadd231ss", "vfmaddsub132pd", "vfmaddsub132ps", "vfmaddsub213pd", "vfmaddsub213ps", "vfmaddsub231pd", "vfmaddsub231ps", "vfmsub132pd", "vfmsub132ps", "vfmsub132sd", "vfmsub132ss", "vfmsub213pd", "vfmsub213ps", "vfmsub213sd", "vfmsub213ss", "vfmsub231pd", "vfmsub231ps", "vfmsub231sd", "vfmsub231ss", "vfmsubadd132pd", "vfmsubadd132ps", "vfmsubadd213pd", "vfmsubadd213ps", "vfmsubadd231pd", "vfmsubadd231ps", "vfnmadd132pd", "vfnmadd132ps", "vfnmadd132sd", "vfnmadd132ss", "vfnmadd213pd", "vfnmadd213ps", "vfnmadd213sd", "vfnmadd213ss", "vfnmadd231pd", "vfnmadd231ps", "vfnmadd231sd", "vfnmadd231ss", "vfnmsub132pd", "vfnmsub132ps", "vfnmsub132sd", "vfnmsub132ss", "vfnmsub213pd", "vfnmsub213ps", "vfnmsub213sd", "vfnmsub213ss", "vfnmsub231pd", "vfnmsub231ps", "vfnmsub231sd", "vfnmsub231ss", "vfpclasspd", "vfpclassps", "vfpclasssd", "vfpclassss", "vgatherdpd", "vgatherdpd", "vgatherdps", "vgatherdps", "vgatherqpd", "vgatherqpd", "vgatherqps", "vgatherqps", "vgetexppd", "vgetexpps", "vgetexpsd", "vgetexpss", "vgetmantpd", "vgetmantps", "vgetmantsd", "vgetmantss", "vinsertf128", "vinsertf32x4", "vinsertf32x8", "vinsertf64x2", "vinsertf64x4", "vinserti128", "vinserti32x4", "vinserti32x8", "vinserti64x2", "vinserti64x4", "vmaskmov", "vmovdqa32", "vmovdqa64", "vmovdqu16", "vmovdqu32", "vmovdqu64", "vmovdqu8", "vpblendd", "vpblendmb", "vpblendmd", "vpblendmq", "vpblendmw", "vpbroadcast", "vpbroadcastb", "vpbroadcastd", "vpbroadcastm", "vpbroadcastq", "vpbroadcastw", "vpcmpb", "vpcmpd", "vpcmpq", "vpcmpub", "vpcmpud", "vpcmpuq", "vpcmpuw", "vpcmpw", "vpcompressd", "vpcompressq", "vpconflictd", "vpconflictq", "vperm2f128", "vperm2i128", "vpermb", "vpermd", "vpermi2b", "vpermi2d", "vpermi2pd", "vpermi2ps", "vpermi2q", "vpermi2w", "vpermilpd", "vpermilps", "vpermpd", "vpermps", "vpermq", "vpermt2b", "vpermt2d", "vpermt2pd", "vpermt2ps", "vpermt2q", "vpermt2w", "vpermw", "vpexpandd", "vpexpandq", "vpgatherdd", "vpgatherdd", "vpgatherdq", "vpgatherdq", "vpgatherqd", "vpgatherqd", "vpgatherqq", "vpgatherqq", "vplzcntd", "vplzcntq", "vpmadd52huq", "vpmadd52luq", "vpmaskmov", "vpmovb2m", "vpmovd2m", "vpmovdb", "vpmovdw", "vpmovm2b", "vpmovm2d", "vpmovm2q", "vpmovm2w", "vpmovq2m", "vpmovqb", "vpmovqd", "vpmovqw", "vpmovsdb", "vpmovsdw", "vpmovsqb", "vpmovsqd", "vpmovsqw", "vpmovswb", "vpmovusdb", "vpmovusdw", "vpmovusqb", "vpmovusqd", "vpmovusqw", "vpmovuswb", "vpmovw2m", "vpmovwb", "vpmultishiftqb", "vprold", "vprolq", "vprolvd", "vprolvq", "vprord", "vprorq", "vprorvd", "vprorvq", "vpscatterdd", "vpscatterdq", "vpscatterqd", "vpscatterqq", "vpsllvd", "vpsllvq", "vpsllvw", "vpsravd", "vpsravq", "vpsravw", "vpsrlvd", "vpsrlvq", "vpsrlvw", "vpternlogd", "vpternlogq", "vptestmb", "vptestmd", "vptestmq", "vptestmw", "vptestnmb", "vptestnmd", "vptestnmq", "vptestnmw", "vrangepd", "vrangeps", "vrangesd", "vrangess", "vrcp14pd", "vrcp14ps", "vrcp14sd", "vrcp14ss", "vreducepd", "vreduceps", "vreducesd", "vreducess", "vrndscalepd", "vrndscaleps", "vrndscalesd", "vrndscaless", "vrsqrt14pd", "vrsqrt14ps", "vrsqrt14sd", "vrsqrt14ss", "vscalefpd", "vscalefps", "vscalefsd", "vscalefss", "vscatterdpd", "vscatterdps", "vscatterqpd", "vscatterqps", "vshuff32x4", "vshuff64x2", "vshufi32x4", "vshufi64x2", "vtestpd", "vtestps", "vzeroall", "vzeroupper", "wait", "wbinvd", "wrfsbase", "wrgsbase", "wrmsr", "wrpkru", "xabort", "xacquire", "xadd", "xbegin", "xchg", "xend", "xgetbv", "xlat", "xlatb", "xor", "xorpd", "xorps", "xrelease", "xrstor", "xrstors", "xsave", "xsavec", "xsaveopt", "xsaves", "xsetbv", "xtest" ] fasm_types = [ "db", "rb", "dw", "rw", "dd", "rd", "dp", "rp", "df", "rf", "dq", "rq", "dt", "rt", "du", ] # Dict where an identifier is assicoated with a string # The string contains characters specifying flags # Available flags: # k - Keyword # m - Macro name # t - fasm data Type name (db, rq, etc.) # s - Struct type name # e - equated constant (name equ value) # = - set constants (name = value) ID_KIND_KEYWORD = 'k' ID_KIND_MACRO_NAME = 'm' ID_KIND_FASM_TYPE = 't' ID_KIND_STRUCT_NAME = 's' ID_KIND_EQUATED_CONSTANT = 'e' ID_KIND_SET_CONSTANT = '=' id2kind = {} # Add kind flag to identifier in id2kind def id_add_kind(identifier, kind): if identifier not in id2kind: id2kind[identifier] = '' id2kind[identifier] += kind # Remove kind flag of identifier in id2kind def id_remove_kind(identifier, kind): if identifier in id2kind: if kind in id2kind[identifier]: id2kind[identifier] = id2kind[identifier].replace(kind, '') # Get kind of an identifier def id_get_kind(identifier): if identifier in id2kind: return id2kind[identifier] else: return '' for keyword in keywords: id_add_kind(keyword, ID_KIND_KEYWORD) for fasm_type in fasm_types: id_add_kind(fasm_type, ID_KIND_FASM_TYPE) # Warning list warnings = "" # Parse arguments parser = argparse.ArgumentParser() parser.add_argument("-o", help="Doxygen output folder") parser.add_argument("--clean", help="Remove generated files", action="store_true") parser.add_argument("--dump", help="Dump all defined symbols", action="store_true") parser.add_argument("--stats", help="Print symbol stats", action="store_true") parser.add_argument("--nowarn", help="Do not write warnings file", action="store_true") parser.add_argument("--noemit", help="Do not emit doxygen files (for testing)", action="store_true") args = parser.parse_args() doxygen_src_path = args.o if args.o else 'docs/doxygen' clean_generated_stuff = args.clean dump_symbols = args.dump print_stats = args.stats enable_warnings = not args.nowarn noemit = args.noemit # Variables, functions, labels, macros, structure types elements = [] class LegacyAsmReader: def __init__(self, file): self.file = file self.lines = open(file, "r", encoding="utf-8").readlines() self.line_idx = 0 self.i = 0 def curr(self): try: return self.lines[self.line_idx][self.i] except: return '' def step(self): c = self.curr() self.i += 1 # Wrap the line if '\\' followed by whitespaces and/or comment while self.curr() == '\\': i_of_backslash = self.i self.i += 1 while self.curr().isspace(): self.i += 1 if self.curr() == ';' or self.curr() == '': self.line_idx += 1 self.i = 0 else: # There's something other than a comment after the backslash # So don't interpret the backslash as a line wrap self.i = i_of_backslash break return c def nextline(self): c = self.curr() while c != '': c = self.step() self.line_idx += 1 self.i = 0 def no_lines(self): if self.line_idx >= len(self.lines): return True return False def location(self): return f"{self.file}:{self.line_idx + 1}" def skip_spaces(self): while self.curr().isspace(): self.step() class AsmReaderRecognizingStrings(LegacyAsmReader): def __init__(self, file): super().__init__(file) self.in_string = None self.should_recognize_strings = True def step(self): c = super().step() if self.should_recognize_strings and (c == '"' or c == "'"): # If just now we was at the double or single quotation mark # and we aren't in a string yet # then say "we are in a string openned with this quotation mark now" if self.in_string == None: self.in_string = c # If just now we was at the double or single quotation mark # and we are in the string entered with the same quotation mark # then say "we aren't in a string anymore" elif self.in_string == c: self.in_string = None return c class AsmReaderReadingComments(AsmReaderRecognizingStrings): def __init__(self, file): super().__init__(file) self.status = dict() self.status_reset() self.comment = '' def status_reset(self): # If the line has non-comment code self.status_has_code = False # If the line has a comment at the end self.status_has_comment = False # Let it recognize strings further, we are definitely out of a comment self.should_recognize_strings = True def status_set_has_comment(self): self.status_has_comment = True # Don't let it recognize strings cause we are in a comment now self.should_recognize_strings = False def status_set_has_code(self): self.status_has_code = True def update_status(self): # If we aren't in a comment and we aren't in a string - say we are now in a comment if ';' met if not self.status_has_comment and not self.in_string and self.curr() == ';': self.status_set_has_comment() # Else if we are in a comment - collect the comment elif self.status_has_comment: self.comment += self.curr() # Else if there's some non-whitespace character out of a comment # then the line has code elif not self.status_has_comment and not self.curr().isspace(): self.status_set_has_code() def step(self): # Get to the next character c = super().step() # Update status of the line according to the next character self.update_status() return c def nextline(self): super().nextline() # If the line we leave was not a comment-only line # then forget the collected comment # Otherwise the collected comment should be complemented by comment from next line in step() if self.status_has_code: self.comment = '' # Reset the line status (now it's the status of the new line) self.status_reset() # Set new status for this line according to the first character in the line self.update_status() class AsmReaderFetchingIdentifiers(AsmReaderReadingComments): def __init__(self, file): super().__init__(file) def fetch_identifier(self): self.skip_spaces() result = '' while is_id(self.curr()): result += self.step() return result class AsmReader(AsmReaderFetchingIdentifiers): def __init__(self, file): super().__init__(file) created_files = [] class AsmElement: def __init__(self, location, name, comment): global warnings # If the element was constructed during this execution then the element is new self.new = True self.location = location self.file = self.location.split(':')[0].replace('\\', '/') self.line = self.location.split(':')[1] self.name = name self.comment = comment if self.comment == '': warnings += f'{self.location}: Undocumented element\n' def dump(self): print(f"{self.comment}") print(f"{self.location}: {self.name}") def emit(self, dest, doxycomment = '', declaration = ''): # Do not emit anything if the symbol is marked as hidden in its comment if '@dont_give_a_doxygen' in self.comment: return global warnings # Redefine default declaration if declaration == '': declaration = f'#define {self.name}' # Check doxycomment if not doxycomment.endswith('\n'): doxycomment += '\n' if doxycomment.split('@brief ')[1][0].islower(): warnings += f"{self.location}: Brief comment starting from lowercase\n" # Build contents to emit contents = '' contents += '/**\n' contents += doxycomment contents += (f"@par Source\n" + f"{self.file}:{self.line}\n") contents += '*/\n' contents += declaration contents += '\n\n' # Get path to file to emit this full_path = dest + '/' + self.file # Remove the file on first access if it was created by previous generation if full_path not in created_files: if os.path.isfile(full_path): os.remove(full_path) created_files.append(full_path) # Create directories need for the file os.makedirs(os.path.dirname(full_path), exist_ok=True) f = open(full_path, "a") contents = ''.join([i if ord(i) < 128 else '?' for i in contents]) f.write(contents) f.close() class AsmVariable(AsmElement): def __init__(self, location, name, comment, type, init): super().__init__(location, name, comment) self.type = type self.init = init def dump(self): super().dump() print(f"Variable") def emit(self, dest): # Build doxycomment specific for the variable doxycomment = '' doxycomment += self.comment if '@brief' not in doxycomment: doxycomment = '@brief ' + doxycomment doxycomment += (f"@par Initial value\n" + f"{self.init}\n") # Build the declaration name = self.name.replace(".", "_") var_type = self.type.replace(".", "_") declaration = f"{var_type} {name};" # Emit this super().emit(dest, doxycomment, declaration) class AsmFunction(AsmElement): def __init__(self, location, name, comment, calling_convention, args, used_regs): super().__init__(location, name, comment) self.calling_convention = calling_convention self.args = args self.used_regs = used_regs def dump(self): super().dump() print(f"Function") def emit(self, dest): # Build doxycomment specific for the variable doxycomment = '' doxycomment += self.comment if '@brief' not in doxycomment: doxycomment = '@brief ' + doxycomment # If there was no arguments, maybe that's just a label # then parse parameters from its comment if len(self.args) == 0 and '@param' in self.comment: i = 0 while '@param' in self.comment[i:]: i = self.comment.index('@param', i) # Skip '@param' i += len('@param') # Skip spaces after '@param' while self.comment[i].isspace(): i += 1 # Get the parameter name name = '' while is_id(self.comment[i]): name += self.comment[i] i += 1 # Save the parameter self.args.append((name, 'arg_t')) # Build the arg list for declaration arg_list = '(' if len(self.args) > 0: argc = 0 for arg in self.args: if argc != 0: arg_list += ", " arg_list += f"{arg[1]} {arg[0]}" argc += 1 arg_list += ')' # Build the declaration name = self.name.replace(".", "_") declaration = f"void {name}{arg_list};" # Emit this super().emit(dest, doxycomment, declaration) class AsmLabel(AsmElement): def __init__(self, location, name, comment): super().__init__(location, name, comment) def dump(self): super().dump() print(f"Label") def emit(self, dest): # Build doxycomment specific for the variable doxycomment = '' doxycomment += self.comment if '@brief' not in doxycomment: doxycomment = '@brief ' + doxycomment # Build the declaration name = self.name.replace(".", "_") declaration = f"label {name};" # Emit this super().emit(dest, doxycomment, declaration) class AsmMacro(AsmElement): def __init__(self, location, name, comment, args): super().__init__(location, name, comment) self.args = args def dump(self): super().dump() print(f"Macro") print(f"Parameters: {self.args}") def emit(self, dest): # Construct arg list without '['s, ']'s and '*'s args = [arg for arg in self.args if arg not in "[]*"] # Construct C-like arg list arg_list = "" if len(args) > 0: arg_list += '(' argc = 0 for arg in args: if argc != 0: arg_list += ", " arg_list += arg argc += 1 arg_list += ')' # Build doxycomment doxycomment = '' doxycomment += self.comment if '@brief' not in doxycomment: doxycomment = '@brief ' + doxycomment # Build declaration declaration = f"#define {self.name}{arg_list}" # Emit this super().emit(dest, doxycomment, declaration) class AsmStruct(AsmElement): def __init__(self, location, name, comment, members): super().__init__(location, name, comment) self.members = members def dump(self): super().dump() print(f"Struct") def emit(self, dest): # Build doxycomment doxycomment = '' doxycomment += self.comment if '@brief' not in doxycomment: doxycomment = '@brief ' + doxycomment doxycomment += '\n' # Build declaration declaration = f"struct {self.name}" + " {\n" for member in self.members: if type(member) == AsmVariable: declaration += f'\t{member.type} {member.name}; /**< {member.comment} */\n' declaration += '};' # Emit this super().emit(dest, doxycomment, declaration) class AsmUnion(AsmElement): def __init__(self, location, name, comment, members): super().__init__(location, name, comment) self.members = members def dump(self): super().dump() print(f"Union") def emit(self, dest): # Build doxycomment doxycomment = '' doxycomment += self.comment if '@brief' not in doxycomment: doxycomment = '@brief ' + doxycomment # Build declaration declaration = f"union {self.name}" + " {};" # Emit this super().emit(dest, doxycomment, declaration) class VariableNameIsMacroName: def __init__(self, name): self.name = name def is_id(c): return c.isprintable() and c not in "+-/*=<>()[]{};:,|&~#`'\" \n\r\t\v" def is_starts_as_id(s): return not s[0].isdigit() def parse_after_macro(r): location = r.location() # Skip spaces after the "macro" keyword r.skip_spaces() # Read macro name name = "" while is_id(r.curr()) or r.curr() == '#': name += r.step() # Skip spaces after macro name r.skip_spaces() # Find all arguments args = [] arg = '' while r.curr() and r.curr() != ';' and r.curr() != '{': # Collect identifier if is_id(r.curr()): arg += r.step() # Save the collected identifier elif r.curr() == ',': args.append(arg) arg = '' r.step() # Just push the '[' elif r.curr() == '[': args.append(r.step()) # Just push the identifier and get ']' ready to be pushed on next comma elif r.curr() == ']': args.append(arg) arg = r.step() # Just push the identifier and get '*' ready to be pushed on next comma elif r.curr() == '*': args.append(arg) arg = r.step() # Just skip whitespaces elif r.curr().isspace(): r.step() # Something unexpected else: raise Exception(f"Unexpected symbol '{r.curr()}' at index #{r.i} " + f"in the macro declaration at {location} " + f"(line: {r.lines[r.line_idx]})\n''") # Append the last argument if arg != '': args.append(arg) # Skip t spaces after the argument list r.skip_spaces() # Get a comment if it is: read till the end of the line and get the comment from the reader while r.curr() != '': r.step() comment = r.comment # Find end of the macro prev = '' while True: if r.curr() == '}' and prev != '\\': break elif r.curr() == '': prev = '' r.nextline() continue prev = r.step() # Build the output return AsmMacro(location, name, comment, args) def parse_variable(r, first_word = None): global warnings location = r.location() # Skip spaces before variable name r.skip_spaces() # Get variable name name = "" # Read it if it was not supplied if first_word == None: while is_id(r.curr()): name += r.step() # Or use the supplied one instead else: name = first_word # Check the name # If it's 0 len, that means threr's something else than an identifier at the beginning if len(name) == 0: return None # If it starts from digit or othervice illegally it's illegal if not is_starts_as_id(name): return None # Get kind of the identifier from id2kind table kind = id_get_kind(name) # If it's a keyword, that's not a variable declaration if ID_KIND_KEYWORD in kind: return None # If it's a macro name, that's not a variable declaration if ID_KIND_MACRO_NAME in kind: return VariableNameIsMacroName(name) # If it's a datatype or a structure name that's not a variable declaration: that's just a data # don't document just a data for now if ID_KIND_STRUCT_NAME in kind or ID_KIND_FASM_TYPE in kind: return None # Skip spaces before type name r.skip_spaces() # Read type name var_type = "" while is_id(r.curr()): var_type += r.step() # Check the type name if len(var_type) == 0: # If there's no type identifier after the name # maybe the name is something meaningful for the next parser # return it return name # If it starts from digit or othervice illegally it's illegal if not is_starts_as_id(var_type): return None # Get kind of type identifier type_kind = id_get_kind(var_type) # If it's a keyword, that's not a variable declaration # return the two words of the lexical structure if ID_KIND_KEYWORD in type_kind: return (name, var_type) # Skip spaces before the value r.skip_spaces() # Read the value until the comment or end of the line value = "" while r.curr() != ';' and r.curr() != '' and r.curr() != '\n': value += r.step() # Skip spaces after the value r.skip_spaces() # Read till end of the line to get a comment from the reader while r.curr() != '': r.step() # Build the result return AsmVariable(location, name, r.comment, var_type, value) def parse_after_struct(r, as_union = True): global warnings location = r.location() # Skip spaces after "struct" keyword r.skip_spaces() # Read struct name name = "" while is_id(r.curr()): name += r.step() # Read till end of the line and get the comment from the reader while r.curr() != '': r.step() comment = r.comment # Get to the next line to parse struct members r.nextline() # Parse struct members members = [] while True: r.skip_spaces() var = parse_variable(r) if type(var) == AsmVariable: members.append(var) elif type(var) == str: if var == 'union': # Parse the union as a struct union = parse_after_struct(r, as_union = True) members.append(union) # Skip the ends of the union r.nextline() elif r.curr() == ':': warnings += f"{r.location()}: Skept the label in the struct\n" else: raise Exception(f"Garbage in struct member at {location} (got '{var}' identifier)") elif type(var) == VariableNameIsMacroName: if var.name == 'ends': break r.nextline() # Return the result if as_union: return AsmStruct(location, name, comment, members) else: return AsmUnion(location, name, comment, members) def parse_after_proc(r): # Get proc name name = r.fetch_identifier() # Next identifier after the proc name identifier = r.fetch_identifier() # Check if the id is 'stdcall' or 'c' (calling convention specifier) # and if so - save the convention and lookup the next identifier calling_convention = '' if identifier == 'stdcall' or identifier == 'c': calling_convention = identifier # If next is a comma, just skip it if r.curr() == ',': r.step() # Read the next identifier identifier = r.fetch_identifier() # Check if the id is 'uses' (used register list specifier) # and if so save the used register list used_regs = [] if identifier == 'uses': # Read the registers while True: reg_name = r.fetch_identifier() if reg_name != '': used_regs.append(reg_name) else: break # If next is a comma, just skip it if r.curr() == ',': r.step() # Read the next identifier identifier = r.fetch_identifier() # Check if there are argument identifiers args = [] while identifier != '': arg_name = identifier arg_type = 'arg_t' # Skip spaces after argument name r.skip_spaces() # If there's a ':' after the name - the next identifier is type if r.curr() == ':': r.step() arg_type = r.fetch_identifier() # If there's a comma - there's one more argument # else no arguments anymore if r.curr() == ',': r.step() identifier = r.fetch_identifier() else: identifier = '' args.append((arg_name, arg_type)) # Get to the end of the line and get a comment from the reader while r.curr() != '': r.step() comment = r.comment # Build the element return AsmFunction(r.location(), name, comment, calling_convention, args, used_regs) def get_declarations(asm_file_contents, asm_file_name): r = AsmReader(asm_file_name) while not r.no_lines(): # Skip leading spaces r.skip_spaces() # Skip the line if it's starting with a comment if r.curr() == ';': r.nextline() continue # Get first word first_word = "" while is_id(r.curr()): first_word += r.step() # Match macro declaration if first_word == "macro": macro = parse_after_macro(r) elements.append(macro) id_add_kind(macro.name, ID_KIND_MACRO_NAME) # Match structure declaration elif first_word == "struct": struct = parse_after_struct(r) elements.append(struct) id_add_kind(struct.name, ID_KIND_STRUCT_NAME) # Match function definition elif first_word == "proc": proc = parse_after_proc(r) elements.append(proc) elif first_word == 'format': # Skip the format directive pass elif first_word == 'include': # Skip the include directive pass elif first_word == 'if': # Skip the conditional directive pass elif first_word == 'repeat': # Skip the repeat directive pass elif first_word == 'purge': while True: # Skip spaces after the 'purge' keyword or after the comma what separated the previous macro name r.skip_spaces() # Get the purged macro name name = '' while is_id(r.curr()): name += r.step() # Remove the purged macro from the macro names list try: id_remove_kind(name, ID_KIND_MACRO_NAME) except: pass # Skip spaces after the name r.skip_spaces() # If it's comma (',') after then that's not the last purged macro, continue purging if r.curr() == ',': r.step() continue # Here we purged all the macros should be purged break # Match label or a variable elif len(first_word) != 0: # Skip spaces after the identifier r.skip_spaces() # Match a variable var = parse_variable(r, first_word) if type(var) == AsmVariable: elements.append(var) # If it wasn't a variable but there was an identifier # Maybe that's a label and the identifier is the label name # The parse_variable returns the first found or supplied identifier # In this case it returns the first_word which is supplied # If it didn't match a type identifier after the word elif type(var) == str: name = var # Match label beginning (':' after name) if r.curr() == ':': # Get to the end of the line and get the coment from the reader while r.curr() != '': r.step() comment = r.comment # Only handle non-local labels if name[0] != '.' and name != "@@" and name != "$Revision": if '@return' in comment or '@param' in comment: element = AsmFunction(r.location(), name, comment, '', [], []) else: element = AsmLabel(r.location(), name, comment) elements.append(element) elif r.curr() == '=': # Save the identifier as a set constant id_add_kind(first_word, ID_KIND_SET_CONSTANT) elif type(var) == tuple: (word_one, word_two) = var if word_two == 'equ': # Save the identifier as an equated constant id_add_kind(word_one, ID_KIND_EQUATED_CONSTANT) r.nextline() def it_neds_to_be_parsed(source_file): # If there's no symbols file saved - parse it anyway # cause we need to create the symbols file and use it # if we gonna generate proper doxygen if not os.path.isfile('asmxygen.elements.pickle'): return True dest = doxygen_src_path + '/' + source_file # If there's no the doxygen file it should be compiled to # then yes, we should compile it to doxygen if not os.path.isfile(dest): return True source_change_time = os.path.getmtime(source_file) dest_change_file = os.path.getmtime(dest) # If the source is newer than the doxygen it was compiled to # then the source should be recompiled (existing doxygen is old) if source_change_time > dest_change_file: return True return False def handle_file(handled_files, asm_file_name, subdir = "."): global elements # Canonicalize the file path and get it relative to cwd cwd = os.path.abspath(os.path.dirname(sys.argv[0])) asm_file_name = os.path.realpath(asm_file_name) asm_file_name = asm_file_name[len(cwd) + 1:] # If it's lang.inc - skip it if asm_file_name == 'lang.inc': return # If the file was handled in this execution before - skip it if asm_file_name in handled_files: return # Say that the file was handled in this execution handled_files.append(asm_file_name) # Check if the file should be parsed (if it was modified or wasn't parsed yet) should_get_declarations = True if not it_neds_to_be_parsed(asm_file_name): print(f"Skipping {asm_file_name} (already newest)") should_get_declarations = False else: print(f"Handling {asm_file_name}") # Remove elements parsed from this file before if any elements_to_remove = [x for x in elements if x.location.split(':')[0] == asm_file_name] elements = [x for x in elements if x.location.split(':')[0] != asm_file_name] # Forget types of identifiers of names of the removed elements for element in elements_to_remove: if type(element) == AsmStruct: id_remove_kind(element.name, ID_KIND_STRUCT_NAME) elif type(element) == AsmMacro: id_remove_kind(element.name, ID_KIND_MACRO_NAME) # Read the source asm_file_contents = open(asm_file_name, "r", encoding="utf-8").read() # Find includes, fix their paths and handle em recoursively includes = re.findall(r'^include (["\'])(.*)\1', asm_file_contents, flags=re.MULTILINE) for include in includes: include = include[1].replace('\\', '/'); full_path = subdir + '/' + include; # If the path isn't valid, maybe that's not relative path if not os.path.isfile(full_path): full_path = include new_subdir = full_path.rsplit('/', 1)[0] handle_file(handled_files, full_path, new_subdir) # Only collect declarations from the file if it wasn't parsed before if should_get_declarations and not clean_generated_stuff: get_declarations(asm_file_contents, asm_file_name) kernel_files = [] # Load remembered list of symbols if os.path.isfile('asmxygen.elements.pickle'): print('Reading existing dump of symbols') (elements, id2kind) = pickle.load(open('asmxygen.elements.pickle', 'rb')) handle_file(kernel_files, "./kernel.asm"); if dump_symbols: for asm_element in elements: asm_element.dump() if clean_generated_stuff: kernel_files_set = set(kernel_files) for file in kernel_files: doxygen_file = f"{doxygen_src_path}/{file}" if (os.path.isfile(doxygen_file)): print(f"Removing {file}... ", end = '') os.remove(doxygen_file) print("Done.") elif not noemit: print(f"Writing doumented sources to {doxygen_src_path}") i = 0 new_elements = [x for x in elements if x.new] for element in new_elements: print(f"[{i + 1}/{len(new_elements)}] Emitting {element.name} from {element.location}") element.emit(doxygen_src_path) i += 1 print(f"Writing dump of symbols to asmxygen.elements.pickle") # Now when the new elements already was written, there's no new elements anymore for element in elements: element.new = False pickle.dump((elements, id2kind), open('asmxygen.elements.pickle', 'wb')) if print_stats: var_count = 0 mac_count = 0 lab_count = 0 fun_count = 0 uni_count = 0 str_count = 0 for element in elements: if type(element) == AsmVariable: var_count += 1 elif type(element) == AsmMacro: mac_count += 1 elif type(element) == AsmLabel: lab_count += 1 elif type(element) == AsmFunction: fun_count += 1 elif type(element) == AsmUnion: uni_count += 1 elif type(element) == AsmStruct: str_count += 1 print(f'Parsed variable count: {var_count}') print(f'Parsed macro count: {mac_count}') print(f'Parsed label count: {lab_count}') print(f'Parsed function count: {fun_count}') print(f'Parsed union type count: {uni_count}') print(f'Parsed structure type count: {str_count}') if enable_warnings: open('asmxygen.txt', "w", encoding = "utf-8").write(warnings)