kolibrios/programs/develop/oberon07/source/AMD64.ob07

(*
    BSD 2-Clause License

    Copyright (c) 2018-2022, Anton Krotov
    All rights reserved.
*)

MODULE AMD64;

IMPORT IL, BIN, WR := WRITER, CHL := CHUNKLISTS, LISTS, PATHS, PROG, TARGETS,
       REG, UTILS, S := STRINGS, PE32, ELF, X86, ERRORS;


CONST

    rax = REG.R0;
    r10 = REG.R10;
    r11 = REG.R11;

    rcx = REG.R1;
    rdx = REG.R2;
    r8  = REG.R8;
    r9  = REG.R9;

    rsp = 4;
    rbp = 5;
    rsi = 6;
    rdi = 7;

    MAX_XMM = 5;

    je = 84H; jne = 85H; jl = 8CH; jge = 8DH; jle = 8EH; jg = 8FH; jb = 82H;

    sete = 94H; setne = 95H; setl = 9CH; setge = 9DH; setle = 9EH; setg = 9FH; setc = 92H; setnc = 93H;

    shl = IL.opLSL2; shr = IL.opLSR2; sar = IL.opASR2; ror = IL.opROR2;

    sCODE = BIN.PICCODE;
    sDATA = BIN.PICDATA;
    sBSS  = BIN.PICBSS;
    sIMP  = BIN.PICIMP;

    FPR_ERR = 41;


TYPE

    COMMAND = IL.COMMAND;

    Number = POINTER TO RECORD (LISTS.ITEM) value: INTEGER END;

    OPRR = PROCEDURE (reg1, reg2: INTEGER);


VAR

    R: REG.REGS;

    Numbers: LISTS.LIST;
    Numbers_Count: INTEGER;
    Numbers_Offs: INTEGER;

    prog: BIN.PROGRAM;

    tcount: INTEGER;

    dllret, sofinit: INTEGER;

    Win64RegPar: ARRAY 4 OF INTEGER;
    SystemVRegPar: ARRAY 6 OF INTEGER;

    Xmm: ARRAY 1000 OF INTEGER;

    fname: PATHS.PATH;


PROCEDURE OutByte (b: BYTE);
BEGIN
    X86.OutByte(b)
END OutByte;


PROCEDURE OutByte2 (a, b: BYTE);
BEGIN
    X86.OutByte(a);
    X86.OutByte(b)
END OutByte2;


PROCEDURE OutByte3 (a, b, c: BYTE);
BEGIN
    X86.OutByte(a);
    X86.OutByte(b);
    X86.OutByte(c)
END OutByte3;


PROCEDURE OutInt (n: INTEGER);
BEGIN
    X86.OutByte(n MOD 256);
    X86.OutByte(UTILS.Byte(n, 1));
    X86.OutByte(UTILS.Byte(n, 2));
    X86.OutByte(UTILS.Byte(n, 3))
END OutInt;


PROCEDURE short (n: INTEGER): INTEGER;
    RETURN 2 * ORD(X86.isByte(n))
END short;


PROCEDURE long (n: INTEGER): INTEGER;
    RETURN 40H * ORD(~X86.isByte(n))
END long;


PROCEDURE OutIntByte (n: INTEGER);
BEGIN
    IF X86.isByte(n) THEN
        OutByte(n MOD 256)
    ELSE
        OutInt(n)
    END
END OutIntByte;


PROCEDURE isLong (n: INTEGER): BOOLEAN;
    RETURN (n > UTILS.max32) OR (n < UTILS.min32)
END isLong;


PROCEDURE NewNumber (value: INTEGER);
VAR
    number: Number;

BEGIN
    NEW(number);
    number.value := value;
    LISTS.push(Numbers, number);
    INC(Numbers_Count)
END NewNumber;


PROCEDURE NewLabel (): INTEGER;
BEGIN
    BIN.NewLabel(prog)
    RETURN IL.NewLabel()
END NewLabel;


PROCEDURE Rex (reg1, reg2: INTEGER);
BEGIN
    OutByte(48H + reg1 DIV 8 + 4 * (reg2 DIV 8))
END Rex;


PROCEDURE lea (reg, offset, section: INTEGER);
BEGIN
    Rex(0, reg);
    OutByte2(8DH, 05H + 8 * (reg MOD 8)); (* lea reg, [rip + offset] *)
    X86.Reloc(section, offset)
END lea;


PROCEDURE oprr (op: BYTE; reg1, reg2: INTEGER); (* op reg1, reg2 *)
BEGIN
    Rex(reg1, reg2);
    OutByte2(op, 0C0H + 8 * (reg2 MOD 8) + reg1 MOD 8)
END oprr;


PROCEDURE oprr2 (op1, op2: BYTE; reg1, reg2: INTEGER); (* op reg1, reg2 *)
BEGIN
    Rex(reg1, reg2);
    OutByte3(op1, op2, 0C0H + 8 * (reg2 MOD 8) + reg1 MOD 8)
END oprr2;


PROCEDURE mov (reg1, reg2: INTEGER); (* mov reg1, reg2 *)
BEGIN
    oprr(89H, reg1, reg2)
END mov;


PROCEDURE xor (reg1, reg2: INTEGER); (* xor reg1, reg2 *)
BEGIN
    oprr(31H, reg1, reg2)
END xor;


PROCEDURE and (reg1, reg2: INTEGER); (* and reg1, reg2 *)
BEGIN
    oprr(21H, reg1, reg2)
END and;


PROCEDURE _or (reg1, reg2: INTEGER); (* or reg1, reg2 *)
BEGIN
    oprr(09H, reg1, reg2)
END _or;


PROCEDURE add (reg1, reg2: INTEGER); (* add reg1, reg2 *)
BEGIN
    oprr(01H, reg1, reg2)
END add;


PROCEDURE sub (reg1, reg2: INTEGER); (* sub reg1, reg2 *)
BEGIN
    oprr(29H, reg1, reg2)
END sub;


PROCEDURE xchg (reg1, reg2: INTEGER); (* xchg reg1, reg2 *)
BEGIN
    IF rax IN {reg1, reg2} THEN
        Rex(reg1 + reg2, 0);
        OutByte(90H + (reg1 + reg2) MOD 8)
    ELSE
        oprr(87H, reg1, reg2)
    END
END xchg;


PROCEDURE cmprr (reg1, reg2: INTEGER);  (* cmp reg1, reg2 *)
BEGIN
    oprr(39H, reg1, reg2)
END cmprr;


PROCEDURE pop (reg: INTEGER); (* pop reg *)
BEGIN
    IF reg >= 8 THEN
        OutByte(41H)
    END;
    OutByte(58H + reg MOD 8)
END pop;


PROCEDURE push (reg: INTEGER); (* push reg *)
BEGIN
    IF reg >= 8 THEN
        OutByte(41H)
    END;
    OutByte(50H + reg MOD 8)
END push;


PROCEDURE decr (reg: INTEGER);
BEGIN
    Rex(reg, 0);
    OutByte2(0FFH, 0C8H + reg MOD 8) (* dec reg1 *)
END decr;


PROCEDURE incr (reg: INTEGER);
BEGIN
    Rex(reg, 0);
    OutByte2(0FFH, 0C0H + reg MOD 8) (* inc reg1 *)
END incr;


PROCEDURE drop;
BEGIN
    REG.Drop(R)
END drop;


PROCEDURE GetAnyReg (): INTEGER;
    RETURN REG.GetAnyReg(R)
END GetAnyReg;


PROCEDURE callimp (label: INTEGER);
BEGIN
    OutByte2(0FFH, 15H);    (* call qword[rip + label + IMP] *)
    X86.Reloc(sIMP, label)
END callimp;


PROCEDURE pushDA (offs: INTEGER);
VAR
    reg: INTEGER;

BEGIN
    reg := GetAnyReg();
    lea(reg, offs, sDATA);
    push(reg);
    drop
END pushDA;


PROCEDURE CallRTL (proc: INTEGER);
VAR
    label: INTEGER;

BEGIN
    label := IL.codes.rtl[proc];
    IF label < 0 THEN
        callimp(-label)
    ELSE
        X86.call(label)
    END
END CallRTL;


PROCEDURE UnOp (VAR reg: INTEGER);
BEGIN
    REG.UnOp(R, reg)
END UnOp;


PROCEDURE BinOp (VAR reg1, reg2: INTEGER);
BEGIN
    REG.BinOp(R, reg1, reg2)
END BinOp;


PROCEDURE PushAll (NumberOfParameters: INTEGER);
BEGIN
    REG.PushAll(R);
    DEC(R.pushed, NumberOfParameters)
END PushAll;


PROCEDURE movabs (reg, n: INTEGER);
VAR
    i: INTEGER;

BEGIN
    Rex(reg, 0);
    OutByte(0B8H + reg MOD 8); (* movabs reg, n *)
    FOR i := 0 TO 7 DO
        OutByte(UTILS.Byte(n, i))
    END
END movabs;


PROCEDURE movrc (reg, n: INTEGER); (* mov reg, n *)
BEGIN
    IF isLong(n) THEN
        movabs(reg, n)
    ELSIF n = 0 THEN
        xor(reg, reg)
    ELSE
        Rex(reg, 0);
        OutByte2(0C7H, 0C0H + reg MOD 8);
        OutInt(n)
    END
END movrc;


PROCEDURE test (reg: INTEGER); (* test reg, reg *)
BEGIN
    oprr(85H, reg, reg)
END test;


PROCEDURE oprlongc (reg, n: INTEGER; oprr: OPRR);
VAR
    reg2: INTEGER;

BEGIN
    reg2 := GetAnyReg();
    ASSERT(reg2 # reg);
    movabs(reg2, n);
    oprr(reg, reg2);
    drop
END oprlongc;


PROCEDURE oprc (op, reg, n: INTEGER; oprr: OPRR);
BEGIN
    IF isLong(n) THEN
        oprlongc(reg, n, oprr)
    ELSE
        Rex(reg, 0);
        X86.oprc(op, reg, n)
    END
END oprc;


PROCEDURE cmprc (reg, n: INTEGER); (* cmp reg, n *)
BEGIN
    IF n = 0 THEN
        test(reg)
    ELSE
        oprc(0F8H, reg, n, cmprr)
    END
END cmprc;


PROCEDURE addrc (reg, n: INTEGER); (* add reg, n *)
BEGIN
    oprc(0C0H, reg, n, add)
END addrc;


PROCEDURE subrc (reg, n: INTEGER); (* sub reg, n *)
BEGIN
    oprc(0E8H, reg, n, sub)
END subrc;


PROCEDURE andrc (reg, n: INTEGER); (* and reg, n *)
BEGIN
    oprc(0E0H, reg, n, and)
END andrc;


PROCEDURE orrc (reg, n: INTEGER); (* or reg, n *)
BEGIN
    oprc(0C8H, reg, n, _or)
END orrc;


PROCEDURE xorrc (reg, n: INTEGER); (* xor reg, n *)
BEGIN
    oprc(0F0H, reg, n, xor)
END xorrc;


PROCEDURE pushc (n: INTEGER);
VAR
    reg2: INTEGER;

BEGIN
    IF isLong(n) THEN
        reg2 := GetAnyReg();
        movabs(reg2, n);
        push(reg2);
        drop
    ELSE
        X86.pushc(n)
    END
END pushc;


PROCEDURE not (reg: INTEGER); (* not reg *)
BEGIN
    Rex(reg, 0);
    OutByte2(0F7H, 0D0H + reg MOD 8)
END not;


PROCEDURE neg (reg: INTEGER); (* neg reg *)
BEGIN
    Rex(reg, 0);
    OutByte2(0F7H, 0D8H + reg MOD 8)
END neg;


PROCEDURE movzx (reg1, reg2, offs: INTEGER; word: BOOLEAN); (* movzx reg1, byte/word[reg2 + offs] *)
BEGIN
    Rex(reg2, reg1);
    X86.movzx(reg1, reg2, offs, word)
END movzx;


PROCEDURE movmr32 (reg1, offs, reg2: INTEGER); (* mov dword[reg1+offs], reg2_32 *)
BEGIN
    X86._movrm(reg2, reg1, offs, 32, TRUE)
END movmr32;


PROCEDURE movrm32 (reg1, reg2, offs: INTEGER); (* mov reg1_32, dword[reg2+offs] *)
BEGIN
    X86._movrm(reg1, reg2, offs, 32, FALSE)
END movrm32;


PROCEDURE movmr (reg1, offs, reg2: INTEGER); (* mov qword[reg1+offs], reg2 *)
BEGIN
    X86._movrm(reg2, reg1, offs, 64, TRUE)
END movmr;


PROCEDURE movrm (reg1, reg2, offs: INTEGER); (* mov reg1, qword[reg2+offs] *)
BEGIN
    X86._movrm(reg1, reg2, offs, 64, FALSE)
END movrm;


PROCEDURE comisd (xmm1, xmm2: INTEGER); (* comisd xmm1, xmm2 *)
BEGIN
    OutByte(66H);
    IF (xmm1 >= 8) OR (xmm2 >= 8) THEN
        OutByte(40H + (xmm1 DIV 8) * 4 + xmm2 DIV 8)
    END;
    OutByte3(0FH, 2FH, 0C0H + (xmm1 MOD 8) * 8 + xmm2 MOD 8)
END comisd;


PROCEDURE _movsdrm (xmm, reg, offs: INTEGER; mr: BOOLEAN);
VAR
    b: BYTE;

BEGIN
    OutByte(0F2H);
    IF (xmm >= 8) OR (reg >= 8) THEN
        OutByte(40H + (xmm DIV 8) * 4 + reg DIV 8)
    END;
    OutByte2(0FH, 10H + ORD(mr));
    IF (offs = 0) & (reg # rbp) THEN
        b := 0
    ELSE
        b := 40H + long(offs)
    END;
    OutByte(b + (xmm MOD 8) * 8 + reg MOD 8);
    IF reg = rsp THEN
        OutByte(24H)
    END;
    IF b # 0 THEN
        OutIntByte(offs)
    END
END _movsdrm;


PROCEDURE movsdrm (xmm, reg, offs: INTEGER); (* movsd xmm, qword[reg+offs] *)
BEGIN
    _movsdrm(xmm, reg, offs, FALSE)
END movsdrm;


PROCEDURE movsdmr (reg, offs, xmm: INTEGER); (* movsd qword[reg+offs], xmm *)
BEGIN
    _movsdrm(xmm, reg, offs, TRUE)
END movsdmr;


PROCEDURE opxx (op, xmm1, xmm2: INTEGER);
BEGIN
    OutByte(0F2H);
    IF (xmm1 >= 8) OR (xmm2 >= 8) THEN
        OutByte(40H + (xmm1 DIV 8) * 4 + xmm2 DIV 8)
    END;
    OutByte3(0FH, op, 0C0H + (xmm1 MOD 8) * 8 + xmm2 MOD 8)
END opxx;


PROCEDURE jcc (cc, label: INTEGER); (* jcc label *)
BEGIN
    X86.jcc(cc, label)
END jcc;


PROCEDURE shiftrc (op, reg, n: INTEGER);
BEGIN
    Rex(reg, 0);
    IF n = 1 THEN
        OutByte(0D1H)
    ELSE
        OutByte(0C1H)
    END;
    X86.shift(op, reg MOD 8);
    IF n # 1 THEN
        OutByte(n)
    END
END shiftrc;


PROCEDURE GetRegA;
BEGIN
    ASSERT(REG.GetReg(R, rax))
END GetRegA;


PROCEDURE Win64Passing (params: INTEGER);
VAR
    n, i: INTEGER;

BEGIN
    n := params MOD 32;
    params := params DIV 32;
    FOR i := 0 TO n - 1 DO
        IF i IN BITS(params) THEN
            movsdrm(i, rsp, i * 8)
        ELSE
            movrm(Win64RegPar[i], rsp, i * 8)
        END
    END
END Win64Passing;


PROCEDURE SysVPassing (params: INTEGER);
VAR
    n, i, s, p, ofs: INTEGER;
    i_count, f_count: INTEGER;
    reg: BOOLEAN;

BEGIN
    ASSERT(r11 IN R.regs);
    n := params MOD 32;
    params := params DIV 32;
    s := 0;

    i_count := 0;
    f_count := 0;
    FOR i := 0 TO n - 1 DO
        IF i IN BITS(params) THEN
            INC(f_count)
        ELSE
            INC(i_count)
        END
    END;

    s := MAX(0, f_count - 8) + MAX(0, i_count - 6);
    p := 0;

    subrc(rsp, s * 8);

    i_count := 0;
    f_count := 0;
    FOR i := 0 TO n - 1 DO
        ofs := (i + s) * 8;
        IF i IN BITS(params) THEN
            reg := f_count <= 7;
            IF reg THEN
                movsdrm(f_count, rsp, ofs);
                INC(f_count)
            END
        ELSE
            reg := i_count <= 5;
            IF reg THEN
                movrm(SystemVRegPar[i_count], rsp, ofs);
                INC(i_count)
            END
        END;

        IF ~reg THEN
            movrm(r11, rsp, ofs);
            movmr(rsp, p, r11);
            INC(p, 8)
        END
    END
END SysVPassing;


PROCEDURE fcmp (op: INTEGER; xmm: INTEGER);
VAR
    cc, reg: INTEGER;

BEGIN
    reg := GetAnyReg();
    xor(reg, reg);
    CASE op OF
    |IL.opEQF:
        comisd(xmm - 1, xmm);
        cc := sete

    |IL.opNEF:
        comisd(xmm - 1, xmm);
        cc := setne

    |IL.opLTF:
        comisd(xmm - 1, xmm);
        cc := setc

    |IL.opGTF:
        comisd(xmm, xmm - 1);
        cc := setc

    |IL.opLEF:
        comisd(xmm, xmm - 1);
        cc := setnc

    |IL.opGEF:
        comisd(xmm - 1, xmm);
        cc := setnc
    END;
    OutByte2(7AH, 3 + reg DIV 8); (* jp L *)
    X86.setcc(cc, reg)
    (* L: *)
END fcmp;


PROCEDURE translate (commands: LISTS.LIST; stroffs: INTEGER);
VAR
    cmd, next: COMMAND;

    opcode, param1, param2, param3, a, b, c, n, label, L, i, cc: INTEGER;

    reg1, reg2, reg3, xmm: INTEGER;

    float: REAL;

BEGIN
    xmm := -1;
    cmd := commands.first(COMMAND);
    WHILE cmd # NIL DO

        param1 := cmd.param1;
        param2 := cmd.param2;

        opcode := cmd.opcode;

        CASE opcode OF

        |IL.opJMP:
            X86.jmp(param1)

        |IL.opCALL, IL.opWIN64CALL, IL.opSYSVCALL:
            CASE opcode OF
            |IL.opCALL:
            |IL.opWIN64CALL: Win64Passing(param2)
            |IL.opSYSVCALL:  SysVPassing(param2)
            END;
            X86.call(param1)

        |IL.opCALLP, IL.opWIN64CALLP, IL.opSYSVCALLP:
            UnOp(reg1);
            IF reg1 # rax THEN
                mov(rax, reg1)
            END;
            drop;
            CASE opcode OF
            |IL.opCALLP:
            |IL.opWIN64CALLP: Win64Passing(param2)
            |IL.opSYSVCALLP:  SysVPassing(param2)
            END;
            OutByte2(0FFH, 0D0H); (* call rax *)
            ASSERT(R.top = -1)

        |IL.opCALLI, IL.opWIN64CALLI, IL.opSYSVCALLI:
            CASE opcode OF
            |IL.opCALLI:
            |IL.opWIN64CALLI: Win64Passing(param2)
            |IL.opSYSVCALLI:  SysVPassing(param2)
            END;
            callimp(param1)

        |IL.opLABEL:
            X86.SetLabel(param1)

        |IL.opERR:
            CallRTL(IL._error)

        |IL.opONERR:
            pushc(param2);
            X86.jmp(param1)

        |IL.opPUSHC:
            pushc(param2)

        |IL.opPRECALL:
            PushAll(0);
            IF (param2 # 0) & (xmm >= 0) THEN
                subrc(rsp, 8)
            END;
            INC(Xmm[0]);
            Xmm[Xmm[0]] := xmm + 1;
            WHILE xmm >= 0 DO
                subrc(rsp, 8);
                movsdmr(rsp, 0, xmm);
                DEC(xmm)
            END;
            ASSERT(xmm = -1)

        |IL.opWIN64ALIGN16:
            ASSERT(rax IN R.regs);
            mov(rax, rsp);
            andrc(rsp, -16);
            push(rax);
            subrc(rsp, (MAX(param2 - 4, 0) MOD 2 + MAX(4 - param2, 0) + 1) * 8)

        |IL.opSYSVALIGN16:
            ASSERT(rax IN R.regs);
            mov(rax, rsp);
            andrc(rsp, -16);
            push(rax);
            IF ~ODD(param2) THEN
                push(rax)
            END

        |IL.opRESF, IL.opRES:
            ASSERT(R.top = -1);
            ASSERT(xmm = -1);
            n := Xmm[Xmm[0]]; DEC(Xmm[0]);

            IF opcode = IL.opRESF THEN
                INC(xmm);
                IF n > 0 THEN
                    movsdmr(rsp, n * 8, 0);
                    DEC(xmm);
                    INC(n)
                END;

                IF xmm + n > MAX_XMM THEN
                    ERRORS.ErrorMsg(fname, param1, param2, FPR_ERR)
                END
            ELSE
                GetRegA
            END;

            WHILE n > 0 DO
                INC(xmm);
                movsdrm(xmm, rsp, 0);
                addrc(rsp, 8);
                DEC(n)
            END

        |IL.opENTER:
            ASSERT(R.top = -1);

            X86.SetLabel(param1);

            param3 := cmd.param3;

            IF param3 > 0 THEN
                push(rbp);
                mov(rbp, rsp);

                n := param3 MOD 32;
                param3 := param3 DIV 32;

                FOR i := 0 TO n - 1 DO
                    IF i IN BITS(param3) THEN
                        movsdmr(rbp, i * 8 + 16, i)
                    ELSE
                        movmr(rbp, i * 8 + 16, Win64RegPar[i])
                    END
                END
            ELSIF param3 < 0 THEN
                param3 := -param3;
                n := (param3 MOD 32) * 8;
                param3 := param3 DIV 32;
                pop(r11);
                subrc(rsp, n);
                push(r11);
                push(rbp);
                mov(rbp, rsp);

                a := 0;
                b := 0;
                c := 0;

                INC(n, 16);

                FOR i := 16 TO n - 8 BY 8 DO
                    IF ODD(param3) THEN
                        IF b <= 7 THEN
                            movsdmr(rbp, i, b);
                            INC(b)
                        ELSE
                            movrm(r11, rbp, n + c);
                            movmr(rbp, i, r11);
                            INC(c, 8)
                        END
                    ELSE
                        IF a <= 5 THEN
                            movmr(rbp, i, SystemVRegPar[a]);
                            INC(a)
                        ELSE
                            movrm(r11, rbp, n + c);
                            movmr(rbp, i, r11);
                            INC(c, 8)
                        END
                    END;
                    param3 := param3 DIV 2
                END
            ELSE
                push(rbp);
                mov(rbp, rsp)
            END;

            n := param2;
            IF n > 4 THEN
                movrc(rcx, n);
                                     (* L: *)
                pushc(0);
                OutByte2(0E2H, 0FCH) (* loop L *)
            ELSE
                WHILE n > 0 DO
                    pushc(0);
                    DEC(n)
                END
            END

        |IL.opLEAVE, IL.opLEAVER, IL.opLEAVEF:
            IF opcode = IL.opLEAVER THEN
                UnOp(reg1);
                IF reg1 # rax THEN
                    mov(rax, reg1)
                END;
                drop
            END;

            ASSERT(R.top = -1);

            IF opcode = IL.opLEAVEF THEN
                DEC(xmm)
            END;

            ASSERT(xmm = -1);

            IF param1 > 0 THEN
                mov(rsp, rbp)
            END;

            pop(rbp);
            IF param2 > 0 THEN
                OutByte3(0C2H, (param2 * 8) MOD 256, (param2 * 8) DIV 256) (* ret param2*8 *)
            ELSE
                X86.ret
            END

        |IL.opSAVES:
            UnOp(reg1);
            REG.PushAll_1(R);
            pushDA(stroffs + param2);
            push(reg1);
            drop;
            pushc(param1);
            CallRTL(IL._move)

        |IL.opSADR:
            lea(GetAnyReg(), stroffs + param2, sDATA)

        |IL.opLOAD8:
            UnOp(reg1);
            movzx(reg1, reg1, 0, FALSE)

        |IL.opLOAD16:
            UnOp(reg1);
            movzx(reg1, reg1, 0, TRUE)

        |IL.opLOAD32:
            UnOp(reg1);
            movrm32(reg1, reg1, 0);
            shiftrc(shl, reg1, 32);
            shiftrc(shr, reg1, 32)

        |IL.opLOAD64:
            UnOp(reg1);
            movrm(reg1, reg1, 0)

        |IL.opLLOAD64:
            reg1 := GetAnyReg();
            movrm(reg1, rbp, param2 * 8)

        |IL.opLLOAD8,
         IL.opLLOAD16:
            reg1 := GetAnyReg();
            movzx(reg1, rbp, param2 * 8, opcode = IL.opLLOAD16)

        |IL.opLLOAD32:
            reg1 := GetAnyReg();
            movrm32(reg1, rbp, param2 * 8);
            shiftrc(shl, reg1, 32);
            shiftrc(shr, reg1, 32)

        |IL.opGLOAD64:
            reg1 := GetAnyReg();
            Rex(0, reg1);  (* mov reg1, qword[rip + param2 + BSS] *)
            OutByte2(8BH, 05H + 8 * (reg1 MOD 8));
            X86.Reloc(sBSS, param2)

        |IL.opGLOAD8, IL.opGLOAD16:
            reg1 := GetAnyReg();
            Rex(0, reg1);  (* movzx reg1, byte/word[rip + param2 + BSS] *)
            OutByte3(0FH, 0B6H + ORD(opcode = IL.opGLOAD16), 05H + 8 * (reg1 MOD 8));
            X86.Reloc(sBSS, param2)

        |IL.opGLOAD32:
            reg1 := GetAnyReg();
            lea(reg1, param2, sBSS);
            movrm32(reg1, reg1, 0);
            shiftrc(shl, reg1, 32);
            shiftrc(shr, reg1, 32)

        |IL.opVLOAD64:
            reg1 := GetAnyReg();
            movrm(reg1, rbp, param2 * 8);
            movrm(reg1, reg1, 0)

        |IL.opVLOAD8,
         IL.opVLOAD16:
            reg1 := GetAnyReg();
            movrm(reg1, rbp, param2 * 8);
            movzx(reg1, reg1, 0, opcode = IL.opVLOAD16)

        |IL.opVLOAD32:
            reg1 := GetAnyReg();
            reg2 := GetAnyReg();
            movrm(reg2, rbp, param2 * 8);
            movrm32(reg1, reg2, 0);
            shiftrc(shl, reg1, 32);
            shiftrc(shr, reg1, 32);
            drop

        |IL.opLADR:
            n := param2 * 8;
            next := cmd.next(COMMAND);
            IF (next.opcode = IL.opSAVEF) OR (next.opcode = IL.opSAVEFI) THEN
                ASSERT(xmm >= 0);
                movsdmr(rbp, n, xmm);
                DEC(xmm);
                cmd := next
            ELSIF next.opcode = IL.opLOADF THEN
                INC(xmm);
                IF xmm > MAX_XMM THEN
                    ERRORS.ErrorMsg(fname, next.param1, next.param2, FPR_ERR)
                END;
                movsdrm(xmm, rbp, n);
                cmd := next
            ELSE
                IF (next.opcode = IL.opADDC) & ~isLong(n + next.param2) THEN
                    INC(n, next.param2);
                    cmd := next
                END;
                reg1 := GetAnyReg();
                Rex(0, reg1);
                OutByte2(8DH, 45H + long(n) + (reg1 MOD 8) * 8); (* lea reg1, qword[rbp+n] *)
                OutIntByte(n)
            END

        |IL.opGADR:
            next := cmd.next(COMMAND);
            IF (next.opcode = IL.opADDC) & ~isLong(param2 + next.param2) THEN
                INC(param2, next.param2);
                cmd := next
            END;
            lea(GetAnyReg(), param2, sBSS)

        |IL.opVADR:
            movrm(GetAnyReg(), rbp, param2 * 8)

        |IL.opSAVE8C:
            UnOp(reg1);
            IF reg1 >= 8 THEN
                OutByte(41H)
            END;
            OutByte3(0C6H, reg1 MOD 8, param2); (* mov byte[reg1], param2 *)
            drop

        |IL.opSAVE16C:
            UnOp(reg1);
            OutByte(66H);
            IF reg1 >= 8 THEN
                OutByte(41H)
            END;
            OutByte2(0C7H, reg1 MOD 8);
            OutByte2(param2 MOD 256, param2 DIV 256); (* mov word[reg1], param2 *)
            drop

        |IL.opSAVEC:
            UnOp(reg1);
            IF isLong(param2) THEN
                reg2 := GetAnyReg();
                movrc(reg2, param2);
                movmr(reg1, 0, reg2);
                drop
            ELSE
                Rex(reg1, 0);
                OutByte2(0C7H, reg1 MOD 8); (* mov qword[reg1], param2 *)
                OutInt(param2)
            END;
            drop

        |IL.opRSET:
            PushAll(2);
            CallRTL(IL._set);
            GetRegA

        |IL.opRSETR:
            PushAll(1);
            pushc(param2);
            CallRTL(IL._set);
            GetRegA

        |IL.opRSETL:
            UnOp(reg1);
            REG.PushAll_1(R);
            pushc(param2);
            push(reg1);
            drop;
            CallRTL(IL._set);
            GetRegA

        |IL.opRSET1:
            PushAll(1);
            CallRTL(IL._set1);
            GetRegA

        |IL.opINCL, IL.opEXCL:
            BinOp(reg1, reg2);
            cmprc(reg1, 64);
            OutByte2(73H, 04H); (* jnb L *)
            Rex(reg2, reg1);
            OutByte3(0FH, 0ABH + 8 * ORD(opcode = IL.opEXCL), 8 * (reg1 MOD 8) + reg2 MOD 8); (* bts/btr qword[reg2], reg1 *)
            (* L: *)
            drop;
            drop

        |IL.opINCLC, IL.opEXCLC:
            UnOp(reg1);
            Rex(reg1, 0);
            OutByte2(0FH, 0BAH); (* bts/btr qword[reg1], param2 *)
            OutByte2(28H + 8 * ORD(opcode = IL.opEXCLC) + reg1 MOD 8, param2);
            drop

        |IL.opEQS .. IL.opGES:
            PushAll(4);
            pushc(opcode - IL.opEQS);
            CallRTL(IL._strcmp);
            GetRegA

        |IL.opEQSW .. IL.opGESW:
            PushAll(4);
            pushc(opcode - IL.opEQSW);
            CallRTL(IL._strcmpw);
            GetRegA

        |IL.opCONST:
            movrc(GetAnyReg(), param2)

        |IL.opEQ..IL.opGE,
         IL.opEQC..IL.opGEC:

            IF (IL.opEQ <= opcode) & (opcode <= IL.opGE) THEN
                BinOp(reg1, reg2);
                cmprr(reg1, reg2);
                drop
            ELSE
                UnOp(reg1);
                cmprc(reg1, param2)
            END;

            drop;
            cc := X86.cond(opcode);

            next := cmd.next(COMMAND);
            IF next.opcode = IL.opJNZ THEN
                jcc(cc, next.param1);
                cmd := next
            ELSIF next.opcode = IL.opJZ THEN
                jcc(X86.inv0(cc), next.param1);
                cmd := next
            ELSE
                reg1 := GetAnyReg();
                X86.setcc(cc + 16, reg1);
                andrc(reg1, 1)
            END

        |IL.opCODE:
            OutByte(param2)

        |IL.opPUSHIP:
            reg1 := GetAnyReg();
            lea(reg1, param2, sIMP);
            movrm(reg1, reg1, 0)

        |IL.opPARAM:
            n := param2;
            IF n = 1 THEN
                UnOp(reg1);
                push(reg1);
                drop
            ELSE
                ASSERT(R.top + 1 <= n);
                PushAll(n)
            END

        |IL.opJNZ1:
            UnOp(reg1);
            test(reg1);
            jcc(jne, param1)

        |IL.opJG:
            UnOp(reg1);
            test(reg1);
            jcc(jg, param1)

        |IL.opJNZ:
            UnOp(reg1);
            test(reg1);
            jcc(jne, param1);
            drop

        |IL.opJZ:
            UnOp(reg1);
            test(reg1);
            jcc(je, param1);
            drop

        |IL.opIN, IL.opINR:
            IF opcode = IL.opINR THEN
                reg2 := GetAnyReg();
                movrc(reg2, param2)
            END;
            label := NewLabel();
            L := NewLabel();
            BinOp(reg1, reg2);
            cmprc(reg1, 64);
            jcc(jb, L);
            xor(reg1, reg1);
            X86.jmp(label);
            X86.SetLabel(L);
            Rex(reg2, reg1);
            OutByte3(0FH, 0A3H, 0C0H + 8 * (reg1 MOD 8) + reg2 MOD 8); (* bt reg2, reg1 *)
            X86.setcc(setc, reg1);
            andrc(reg1, 1);
            X86.SetLabel(label);
            drop

        |IL.opINL:
            UnOp(reg1);
            Rex(reg1, 0);
            OutByte2(0FH, 0BAH); (* bt reg1, param2 *)
            OutByte2(0E0H + reg1 MOD 8, param2);
            X86.setcc(setc, reg1);
            andrc(reg1, 1)

        |IL.opNOT:
            UnOp(reg1);
            test(reg1);
            X86.setcc(sete, reg1);
            andrc(reg1, 1)

        |IL.opORD:
            UnOp(reg1);
            test(reg1);
            X86.setcc(setne, reg1);
            andrc(reg1, 1)

        |IL.opABS:
            UnOp(reg1);
            test(reg1);
            OutByte2(7DH, 03H); (* jge L *)
            neg(reg1)
            (* L: *)

        |IL.opEQB, IL.opNEB:
            BinOp(reg1, reg2);
            drop;
            test(reg1);
            label := NewLabel();
            jcc(je, label);
            movrc(reg1, 1);
            X86.SetLabel(label);
            test(reg2);
            label := NewLabel();
            jcc(je, label);
            movrc(reg2, 1);
            X86.SetLabel(label);
            cmprr(reg1, reg2);
            IF opcode = IL.opEQB THEN
                X86.setcc(sete, reg1)
            ELSE
                X86.setcc(setne, reg1)
            END;
            andrc(reg1, 1)

        |IL.opMULSC:
            UnOp(reg1);
            andrc(reg1, param2)

        |IL.opDIVSC:
            UnOp(reg1);
            xorrc(reg1, param2)

        |IL.opADDSC:
            UnOp(reg1);
            orrc(reg1, param2)

        |IL.opSUBSL:
            UnOp(reg1);
            not(reg1);
            andrc(reg1, param2)

        |IL.opSUBSR:
            UnOp(reg1);
            andrc(reg1, ORD(-BITS(param2)))

        |IL.opMULS:
            BinOp(reg1, reg2);
            and(reg1, reg2);
            drop

        |IL.opDIVS:
            BinOp(reg1, reg2);
            xor(reg1, reg2);
            drop

        |IL.opUMINS:
            UnOp(reg1);
            not(reg1)

        |IL.opCOPY:
            IF (0 < param2) & (param2 <= 256) THEN
                BinOp(reg1, reg2);
                reg3 := GetAnyReg();
                FOR n := 0 TO param2 - param2 MOD 8 - 1 BY 8 DO
                    movrm(reg3, reg1, n);
                    movmr(reg2, n, reg3)
                END;
                n := param2 - param2 MOD 8;
                IF param2 MOD 8 >= 4 THEN
                    movrm32(reg3, reg1, n);
                    movmr32(reg2, n, reg3);
                    INC(n, 4);
                    DEC(param2, 4)
                END;
                IF param2 MOD 8 >= 2 THEN
                    X86.movrm16(reg3, reg1, n);
                    X86.movmr16(reg2, n, reg3);
                    INC(n, 2);
                    DEC(param2, 2)
                END;
                IF param2 MOD 8 = 1 THEN
                    X86.movrm8(reg3, reg1, n);
                    X86.movmr8(reg2, n, reg3);
                END;
                drop;
                drop;
                drop
            ELSE
                PushAll(2);
                pushc(param2);
                CallRTL(IL._move)
            END

        |IL.opMOVE:
            PushAll(3);
            CallRTL(IL._move)

        |IL.opCOPYA:
            PushAll(4);
            pushc(param2);
            CallRTL(IL._arrcpy);
            GetRegA

        |IL.opCOPYS:
            PushAll(4);
            pushc(param2);
            CallRTL(IL._strcpy)

        |IL.opROT:
            PushAll(0);
            push(rsp);
            pushc(param2);
            CallRTL(IL._rot)

        |IL.opNEW:
            PushAll(1);
            n := param2 + 8;
            ASSERT(UTILS.Align(n, 8));
            pushc(n);
            pushc(param1);
            CallRTL(IL._new)

        |IL.opDISP:
            PushAll(1);
            CallRTL(IL._dispose)

        |IL.opPUSHT:
            UnOp(reg1);
            movrm(GetAnyReg(), reg1, -8)

        |IL.opISREC:
            PushAll(2);
            pushc(param2 * tcount);
            CallRTL(IL._isrec);
            GetRegA

        |IL.opIS:
            PushAll(1);
            pushc(param2 * tcount);
            CallRTL(IL._is);
            GetRegA

        |IL.opTYPEGR:
            PushAll(1);
            pushc(param2 * tcount);
            CallRTL(IL._guardrec);
            GetRegA

        |IL.opTYPEGP:
            UnOp(reg1);
            PushAll(0);
            push(reg1);
            pushc(param2 * tcount);
            CallRTL(IL._guard);
            GetRegA

        |IL.opTYPEGD:
            UnOp(reg1);
            PushAll(0);
            X86.pushm(reg1, -8);
            pushc(param2 * tcount);
            CallRTL(IL._guardrec);
            GetRegA

        |IL.opCASET:
            push(rcx);
            push(rcx);
            pushc(param2 * tcount);
            CallRTL(IL._guardrec);
            pop(rcx);
            test(rax);
            jcc(jne, param1)

        |IL.opSAVEP:
            UnOp(reg1);
            reg2 := GetAnyReg();
            lea(reg2, param2, sCODE);
            movmr(reg1, 0, reg2);
            drop;
            drop

        |IL.opPUSHP:
            lea(GetAnyReg(), param2, sCODE)

        |IL.opINC, IL.opDEC:
            BinOp(reg1, reg2);
            (* add/sub qword[reg2], reg1 *)
            Rex(reg2, reg1);
            OutByte2(01H + 28H * ORD(opcode = IL.opDEC), reg2 MOD 8 + (reg1 MOD 8) * 8);
            drop;
            drop

        |IL.opINCC:
            UnOp(reg1);
            IF isLong(param2) THEN
                reg2 := GetAnyReg();
                movrc(reg2, param2);
                (* add qword[reg1], reg2 *)
                Rex(reg1, reg2);
                OutByte2(01H, reg1 MOD 8 + (reg2 MOD 8) * 8);
                drop
            ELSIF ABS(param2) = 1 THEN
                Rex(reg1, 0);
                OutByte2(0FFH, reg1 MOD 8 + 8 * ORD(param2 = -1)) (* inc/dec qword[reg1] *)
            ELSE
                (* add qword[reg1], param2 *)
                Rex(reg1, 0);
                OutByte2(81H + short(param2), reg1 MOD 8);
                OutIntByte(param2)
            END;
            drop

        |IL.opDROP:
            UnOp(reg1);
            drop

        |IL.opSAVE, IL.opSAVE64:
            BinOp(reg2, reg1);
            movmr(reg1, 0, reg2);
            drop;
            drop

        |IL.opSAVE8:
            BinOp(reg2, reg1);
            X86.movmr8(reg1, 0, reg2);
            drop;
            drop

        |IL.opSAVE16:
            BinOp(reg2, reg1);
            X86.movmr16(reg1, 0, reg2);
            drop;
            drop

        |IL.opSAVE32:
            BinOp(reg2, reg1);
            movmr32(reg1, 0, reg2);
            drop;
            drop

        |IL.opMAX, IL.opMIN:
            BinOp(reg1, reg2);
            cmprr(reg1, reg2);
            OutByte2(7DH + ORD(opcode = IL.opMIN), 3); (* jge/jle L *)
            mov(reg1, reg2);
            (* L: *)
            drop

        |IL.opMAXC, IL.opMINC:
            UnOp(reg1);
            cmprc(reg1, param2);
            label := NewLabel();
            IF opcode = IL.opMINC THEN
                cc := jle
            ELSE
                cc := jge
            END;
            jcc(cc, label);
            movrc(reg1, param2);
            X86.SetLabel(label)

        |IL.opSBOOL:
            BinOp(reg2, reg1);
            test(reg2);
            IF reg1 >= 8 THEN
                OutByte(41H)
            END;
            OutByte3(0FH, 95H, reg1 MOD 8); (* setne byte[reg1] *)
            drop;
            drop

        |IL.opSBOOLC:
            UnOp(reg1);
            IF reg1 >= 8 THEN
                OutByte(41H)
            END;
            OutByte3(0C6H, reg1 MOD 8, ORD(param2 # 0)); (* mov byte[reg1], 0/1 *)
            drop

        |IL.opUMINUS:
            UnOp(reg1);
            neg(reg1)

        |IL.opADD:
            BinOp(reg1, reg2);
            add(reg1, reg2);
            drop

        |IL.opSUB:
            BinOp(reg1, reg2);
            sub(reg1, reg2);
            drop

        |IL.opSUBR, IL.opSUBL:
            UnOp(reg1);
            IF param2 = 1 THEN
                decr(reg1)
            ELSIF param2 = -1 THEN
                incr(reg1)
            ELSIF param2 # 0 THEN
                subrc(reg1, param2)
            END;
            IF opcode = IL.opSUBL THEN
                neg(reg1)
            END

        |IL.opADDC:
            IF (param2 # 0) & ~isLong(param2) THEN
                UnOp(reg1);
                next := cmd.next(COMMAND);
                CASE next.opcode OF
                |IL.opLOAD64:
                    movrm(reg1, reg1, param2);
                    cmd := next
                |IL.opLOAD32:
                    movrm32(reg1, reg1, param2);
                    shiftrc(shl, reg1, 32);
                    shiftrc(shr, reg1, 32);
                    cmd := next
                |IL.opLOAD16:
                    movzx(reg1, reg1, param2, TRUE);
                    cmd := next
                |IL.opLOAD8:
                    movzx(reg1, reg1, param2, FALSE);
                    cmd := next
                |IL.opLOAD64_PARAM:
                    X86.pushm(reg1, param2);
                    drop;
                    cmd := next
                ELSE
                    IF param2 = 1 THEN
                        incr(reg1)
                    ELSIF param2 = -1 THEN
                        decr(reg1)
                    ELSE
                        addrc(reg1, param2)
                    END
                END
            ELSIF isLong(param2) THEN
                UnOp(reg1);
                addrc(reg1, param2)
            END

        |IL.opDIV:
            PushAll(2);
            CallRTL(IL._divmod);
            GetRegA

        |IL.opDIVR:
            n := UTILS.Log2(param2);
            IF n > 0 THEN
                UnOp(reg1);
                shiftrc(sar, reg1, n)
            ELSIF n < 0 THEN
                PushAll(1);
                pushc(param2);
                CallRTL(IL._divmod);
                GetRegA
            END

        |IL.opDIVL:
            UnOp(reg1);
            REG.PushAll_1(R);
            pushc(param2);
            push(reg1);
            drop;
            CallRTL(IL._divmod);
            GetRegA

        |IL.opMOD:
            PushAll(2);
            CallRTL(IL._divmod);
            mov(rax, rdx);
            GetRegA

        |IL.opMODR:
            n := UTILS.Log2(param2);
            IF n > 0 THEN
                UnOp(reg1);
                andrc(reg1, param2 - 1);
            ELSIF n < 0 THEN
                PushAll(1);
                pushc(param2);
                CallRTL(IL._divmod);
                mov(rax, rdx);
                GetRegA
            ELSE
                UnOp(reg1);
                xor(reg1, reg1)
            END

        |IL.opMODL:
            UnOp(reg1);
            REG.PushAll_1(R);
            pushc(param2);
            push(reg1);
            drop;
            CallRTL(IL._divmod);
            mov(rax, rdx);
            GetRegA

        |IL.opMUL:
            BinOp(reg1, reg2);
            oprr2(0FH, 0AFH, reg2, reg1); (* imul reg1, reg2 *)
            drop

        |IL.opMULC:
            IF (cmd.next(COMMAND).opcode = IL.opADD) & ((param2 = 2) OR (param2 = 4) OR (param2 = 8)) THEN
                BinOp(reg1, reg2);
                OutByte2(48H + 5 * (reg1 DIV 8) + 2 * (reg2 DIV 8), 8DH); (* lea reg1, [reg1 + reg2 * param2] *)
                reg1 := reg1 MOD 8;
                reg2 := reg2 MOD 8;
                OutByte2(04H + reg1 * 8, reg1 + reg2 * 8 + 40H * UTILS.Log2(param2));
                drop;
                cmd := cmd.next(COMMAND)
            ELSE
                UnOp(reg1);

                a := param2;
                IF a > 1 THEN
                    n := UTILS.Log2(a)
                ELSIF a < -1 THEN
                    n := UTILS.Log2(-a)
                ELSE
                    n := -1
                END;

                IF a = 1 THEN

                ELSIF a = -1 THEN
                    neg(reg1)
                ELSIF a = 0 THEN
                    xor(reg1, reg1)
                ELSE
                    IF n > 0 THEN
                        IF a < 0 THEN
                            neg(reg1)
                        END;
                        shiftrc(shl, reg1, n)
                    ELSE
                        IF isLong(a) THEN
                            reg2 := GetAnyReg();
                            movabs(reg2, a);
                            ASSERT(reg1 # reg2);
                            oprr2(0FH, 0AFH, reg2, reg1); (* imul reg1, reg2 *)
                            drop
                        ELSE
                            (* imul reg1, a *)
                            Rex(reg1, reg1);
                            OutByte2(69H + short(a), 0C0H + (reg1 MOD 8) * 9);
                            OutIntByte(a)
                        END
                    END
                END
            END

        |IL.opADDS:
            BinOp(reg1, reg2);
            _or(reg1, reg2);
            drop

        |IL.opSUBS:
            BinOp(reg1, reg2);
            not(reg2);
            and(reg1, reg2);
            drop

        |IL.opNOP, IL.opAND, IL.opOR:

        |IL.opSWITCH:
            UnOp(reg1);
            IF param2 = 0 THEN
                reg2 := rax
            ELSE
                reg2 := rcx
            END;
            IF reg1 # reg2 THEN
                ASSERT(REG.GetReg(R, reg2));
                ASSERT(REG.Exchange(R, reg1, reg2));
                drop
            END;
            drop

        |IL.opENDSW:

        |IL.opCASEL:
            GetRegA;
            cmprc(rax, param1);
            jcc(jl, param2);
            drop

        |IL.opCASER:
            GetRegA;
            cmprc(rax, param1);
            jcc(jg, param2);
            drop

        |IL.opCASELR:
            GetRegA;
            cmprc(rax, param1);
            IF param2 = cmd.param3 THEN
                jcc(jne, param2)
            ELSE
                jcc(jl, param2);
                jcc(jg, cmd.param3)
            END;
            drop

        |IL.opASR, IL.opROR, IL.opLSL, IL.opLSR:
            UnOp(reg1);
            IF reg1 # rcx THEN
                ASSERT(REG.GetReg(R, rcx));
                ASSERT(REG.Exchange(R, reg1, rcx));
                drop
            END;

            BinOp(reg1, reg2);
            ASSERT(reg2 = rcx);
            Rex(reg1, 0);
            OutByte(0D3H);
            X86.shift(opcode, reg1 MOD 8); (* shift reg1, cl *)
            drop

        |IL.opASR1, IL.opROR1, IL.opLSL1, IL.opLSR1:
            UnOp(reg1);
            IF reg1 # rcx THEN
                ASSERT(REG.GetReg(R, rcx));
                ASSERT(REG.Exchange(R, reg1, rcx));
                drop
            END;

            reg1 := GetAnyReg();
            movrc(reg1, param2);
            BinOp(reg1, reg2);
            ASSERT(reg1 = rcx);
            Rex(reg2, 0);
            OutByte(0D3H);
            X86.shift(opcode, reg2 MOD 8); (* shift reg2, cl *)
            drop;
            drop;
            ASSERT(REG.GetReg(R, reg2))

        |IL.opASR2, IL.opROR2, IL.opLSL2, IL.opLSR2:
            UnOp(reg1);
            shiftrc(opcode, reg1, param2 MOD 64)

        |IL.opGET, IL.opGETC:
            IF opcode = IL.opGET THEN
                BinOp(reg1, reg2)
            ELSIF opcode = IL.opGETC THEN
                UnOp(reg2);
                reg1 := GetAnyReg();
                movrc(reg1, param1)
            END;
            drop;
            drop;
            X86._movrm(reg1, reg1, 0, param2 * 8, FALSE);
            X86._movrm(reg1, reg2, 0, param2 * 8, TRUE)

        |IL.opCHKBYTE:
            BinOp(reg1, reg2);
            cmprc(reg1, 256);
            jcc(jb, param1)

        |IL.opCHKIDX:
            UnOp(reg1);
            cmprc(reg1, param2);
            jcc(jb, param1)

        |IL.opCHKIDX2:
            BinOp(reg1, reg2);
            IF param2 # -1 THEN
                cmprr(reg2, reg1);
                jcc(jb, param1);
            END;
            INCL(R.regs, reg1);
            DEC(R.top);
            R.stk[R.top] := reg2

        |IL.opLENGTH:
            PushAll(2);
            CallRTL(IL._length);
            GetRegA

        |IL.opLENGTHW:
            PushAll(2);
            CallRTL(IL._lengthw);
            GetRegA

        |IL.opLEN:
            n := param2;
            UnOp(reg1);
            drop;
            EXCL(R.regs, reg1);

            WHILE n > 0 DO
                UnOp(reg2);
                drop;
                DEC(n)
            END;

            INCL(R.regs, reg1);
            ASSERT(REG.GetReg(R, reg1))

        |IL.opCHR:
            UnOp(reg1);
            andrc(reg1, 255)

        |IL.opWCHR:
            UnOp(reg1);
            andrc(reg1, 65535)

        |IL.opEQP, IL.opNEP, IL.opEQIP, IL.opNEIP:
            UnOp(reg1);
            reg2 := GetAnyReg();

            CASE opcode OF
            |IL.opEQP, IL.opNEP:
                lea(reg2, param1, sCODE)

            |IL.opEQIP, IL.opNEIP:
                lea(reg2, param1, sIMP);
                movrm(reg2, reg2, 0)
            END;

            cmprr(reg1, reg2);
            drop;
            drop;
            reg1 := GetAnyReg();

            CASE opcode OF
            |IL.opEQP, IL.opEQIP: X86.setcc(sete,  reg1)
            |IL.opNEP, IL.opNEIP: X86.setcc(setne, reg1)
            END;

            andrc(reg1, 1)

        |IL.opINCCB, IL.opDECCB:
            UnOp(reg1);
            IF reg1 >= 8 THEN
                OutByte(41H)
            END;
            OutByte3(80H, 28H * ORD(opcode = IL.opDECCB) + reg1 MOD 8, param2 MOD 256); (* add/sub byte[reg1], param2 MOD 256 *)
            drop

        |IL.opINCB, IL.opDECB:
            BinOp(reg1, reg2);
            IF (reg1 >= 8) OR (reg2 >= 8) THEN
                OutByte(40H + reg2 DIV 8 + 4 * (reg1 DIV 8))
            END;
            OutByte2(28H * ORD(opcode = IL.opDECB), reg2 MOD 8 + 8 * (reg1 MOD 8)); (* add/sub byte[reg2], reg1_8 *)
            drop;
            drop

        |IL.opSAVEIP:
            UnOp(reg1);
            reg2 := GetAnyReg();
            lea(reg2, param2, sIMP);
            movrm(reg2, reg2, 0);
            push(reg2);
            drop;
            IF reg1 >= 8 THEN
                OutByte(41H)
            END;
            OutByte2(8FH, reg1 MOD 8); (* pop qword[reg1] *)
            drop

        |IL.opCLEANUP:
            IF param2 # 0 THEN
                addrc(rsp, param2 * 8)
            END

        |IL.opPOPSP:
            pop(rsp)

        |IL.opLOADF:
            UnOp(reg1);
            INC(xmm);
            IF xmm > MAX_XMM THEN
                ERRORS.ErrorMsg(fname, param1, param2, FPR_ERR)
            END;
            movsdrm(xmm, reg1, 0);
            drop

        |IL.opPUSHF:
            ASSERT(xmm >= 0);
            subrc(rsp, 8);
            movsdmr(rsp, 0, xmm);
            DEC(xmm)

        |IL.opCONSTF:
            float := cmd.float;
            INC(xmm);
            IF xmm > MAX_XMM THEN
                ERRORS.ErrorMsg(fname, param1, param2, FPR_ERR)
            END;
            (* movsd xmm, qword ptr [rip + Numbers_Offs + Numbers_Count * 8 + DATA] *)
            OutByte(0F2H);
            IF xmm >= 8 THEN
                OutByte(44H)
            END;
            OutByte3(0FH, 10H, 05H + 8 * (xmm MOD 8));
            X86.Reloc(sDATA, Numbers_Offs + Numbers_Count * 8);
            NewNumber(UTILS.splitf(float, a, b))

        |IL.opSAVEF, IL.opSAVEFI:
            ASSERT(xmm >= 0);
            UnOp(reg1);
            movsdmr(reg1, 0, xmm);
            DEC(xmm);
            drop

        |IL.opADDF:
            ASSERT(xmm >= 1);
            opxx(58H, xmm - 1, xmm);
            DEC(xmm)

        |IL.opSUBF:
            ASSERT(xmm >= 1);
            opxx(5CH, xmm - 1, xmm);
            DEC(xmm)

        |IL.opSUBFI:
            ASSERT(xmm >= 1);
            opxx(5CH, xmm, xmm - 1);
            opxx(10H, xmm - 1, xmm);
            DEC(xmm)

        |IL.opMULF:
            ASSERT(xmm >= 1);
            opxx(59H, xmm - 1, xmm);
            DEC(xmm)

        |IL.opDIVF:
            ASSERT(xmm >= 1);
            opxx(5EH, xmm - 1, xmm);
            DEC(xmm)

        |IL.opDIVFI:
            ASSERT(xmm >= 1);
            opxx(5EH, xmm, xmm - 1);
            opxx(10H, xmm - 1, xmm);
            DEC(xmm)

        |IL.opFABS, IL.opUMINF: (* andpd/xorpd xmm, xmmword[rip + Numbers_Offs + (16) + DATA] *)
            ASSERT(xmm >= 0);
            OutByte(66H);
            IF xmm >= 8 THEN
                OutByte(44H)
            END;
            OutByte3(0FH, 54H + 3 * ORD(opcode = IL.opUMINF), 05H + (xmm MOD 8) * 8);
            X86.Reloc(sDATA, Numbers_Offs + 16 * ORD(opcode = IL.opFABS))

        |IL.opFLT:
            UnOp(reg1);
            INC(xmm);
            IF xmm > MAX_XMM THEN
                ERRORS.ErrorMsg(fname, param1, param2, FPR_ERR)
            END;
            OutByte(0F2H); Rex(reg1, xmm); OutByte(0FH); (* cvtsi2sd xmm, reg1 *)
            OutByte2(2AH, 0C0H + (xmm MOD 8) * 8 + reg1 MOD 8);
            drop

        |IL.opFLOOR:
            ASSERT(xmm >= 0);
            reg1 := GetAnyReg();
            subrc(rsp, 8);
            OutByte3(00FH, 0AEH, 05CH); OutByte2(024H, 004H);                       (* stmxcsr dword[rsp+4];                              *)
            OutByte2(00FH, 0AEH); OutByte2(01CH, 024H);                             (* stmxcsr dword[rsp];                                *)
            OutByte3(081H, 024H, 024H); OutByte2(0FFH, 09FH); OutByte2(0FFH, 0FFH); (* and dword[rsp],11111111111111111001111111111111b;  *)
            OutByte3(081H, 00CH, 024H); OutByte2(000H, 020H); OutByte2(000H, 000H); (* or dword[rsp],00000000000000000010000000000000b;   *)
            OutByte2(00FH, 0AEH); OutByte2(014H, 024H);                             (* ldmxcsr dword[rsp];                                *)
            OutByte(0F2H); Rex(xmm, reg1); OutByte(0FH);                            (* cvtsd2si reg1, xmm                                 *)
            OutByte2(2DH, 0C0H + xmm MOD 8 + (reg1 MOD 8) * 8);
            OutByte3(00FH, 0AEH, 054H); OutByte2(024H, 004H);                       (* ldmxcsr dword[rsp+4];                              *)
            addrc(rsp, 8);
            DEC(xmm)

        |IL.opEQF .. IL.opGEF:
            ASSERT(xmm >= 1);
            fcmp(opcode, xmm);
            DEC(xmm, 2)

        |IL.opINF:
            INC(xmm);
            IF xmm > MAX_XMM THEN
                ERRORS.ErrorMsg(fname, param1, param2, FPR_ERR)
            END;
            (* movsd xmm, qword ptr [rip + Numbers_Offs + 32 + DATA] *)
            OutByte(0F2H);
            IF xmm >= 8 THEN
                OutByte(44H)
            END;
            OutByte3(0FH, 10H, 05H + 8 * (xmm MOD 8));
            X86.Reloc(sDATA, Numbers_Offs + 32)

        |IL.opPACK, IL.opPACKC:
            IF opcode = IL.opPACK THEN
                BinOp(reg1, reg2)
            ELSE
                UnOp(reg1);
                reg2 := GetAnyReg();
                movrc(reg2, param2)
            END;
            push(reg1);
            movrm(reg1, reg1, 0);
            shiftrc(shl, reg1, 1);
            shiftrc(shr, reg1, 53);
            add(reg1, reg2);
            andrc(reg1, ORD({0..10}));
            shiftrc(shl, reg1, 52);
            movrm(reg2, rsp, 0);
            movrm(reg2, reg2, 0);

            push(reg1);
            lea(reg1, Numbers_Offs + 40, sDATA); (* {0..51, 63} *)
            movrm(reg1, reg1, 0);
            and(reg2, reg1);
            pop(reg1);

            _or(reg2, reg1);
            pop(reg1);
            movmr(reg1, 0, reg2);
            drop;
            drop

        |IL.opUNPK, IL.opLADR_UNPK:

            IF opcode = IL.opLADR_UNPK THEN
                n := param2 * 8;
                UnOp(reg1);
                reg2 := GetAnyReg();
                Rex(0, reg2);
                OutByte2(8DH, 45H + long(n) + (reg2 MOD 8) * 8); (* lea reg2, qword[rbp+n] *)
                OutIntByte(n)
            ELSE
                BinOp(reg1, reg2)
            END;

            push(reg1);
            movrm(reg1, reg1, 0);
            shiftrc(shl, reg1, 1);
            shiftrc(shr, reg1, 53);
            subrc(reg1, 1023);

            movmr(reg2, 0, reg1);

            pop(reg2);
            movrm(reg1, reg2, 0);

            push(reg2);
            lea(reg2, Numbers_Offs + 48, sDATA); (* {52..61} *)
            movrm(reg2, reg2, 0);
            _or(reg1, reg2);
            pop(reg2);

            Rex(reg1, 0);
            OutByte2(0FH, 0BAH);
            OutByte2(0F0H + reg1 MOD 8, 3EH); (* btr reg1, 62 *)
            movmr(reg2, 0, reg1);
            drop;
            drop

        |IL.opSADR_PARAM:
            pushDA(stroffs + param2)

        |IL.opVADR_PARAM:
            X86.pushm(rbp, param2 * 8)

        |IL.opLOAD64_PARAM:
            UnOp(reg1);
            X86.pushm(reg1, 0);
            drop

        |IL.opLLOAD64_PARAM:
            X86.pushm(rbp, param2 * 8)

        |IL.opGLOAD64_PARAM:
            OutByte2(0FFH, 35H); (* push qword[rip + param2 + BSS] *)
            X86.Reloc(sBSS, param2)

        |IL.opCONST_PARAM:
            pushc(param2)

        |IL.opGLOAD32_PARAM, IL.opLOAD32_PARAM:
            IF opcode = IL.opGLOAD32_PARAM THEN
                reg1 := GetAnyReg();
                lea(reg1, param2, sBSS)
            ELSE
                UnOp(reg1)
            END;
            movrm32(reg1, reg1, 0);
            shiftrc(shl, reg1, 32);
            shiftrc(shr, reg1, 32);
            push(reg1);
            drop

        |IL.opLLOAD32_PARAM:
            reg1 := GetAnyReg();
            movrm32(reg1, rbp, param2 * 8);
            shiftrc(shl, reg1, 32);
            shiftrc(shr, reg1, 32);
            push(reg1);
            drop

        |IL.opLADR_SAVEC:
            n := param1 * 8;
            IF isLong(param2) THEN
                reg2 := GetAnyReg();
                movrc(reg2, param2);
                movmr(rbp, n, reg2);
                drop
            ELSE
                OutByte3(48H, 0C7H, 45H + long(n)); (* mov qword[rbp+n], param2 *)
                OutIntByte(n);
                OutInt(param2)
            END

        |IL.opGADR_SAVEC:
            IF isLong(param2) THEN
                reg1 := GetAnyReg();
                movrc(reg1, param2);
                reg2 := GetAnyReg();
                lea(reg2, param1, sBSS);
                movmr(reg2, 0, reg1);
                drop;
                drop
            ELSE
                (* mov qword[rip + param1 - 4 + BSS], param2 *)
                OutByte3(48H, 0C7H, 05H);
                X86.Reloc(sBSS, param1 - 4);
                OutInt(param2)
            END

        |IL.opLADR_SAVE:
            UnOp(reg1);
            movmr(rbp, param2 * 8, reg1);
            drop

        |IL.opLADR_INCC:
            IF isLong(param2) THEN
                reg2 := GetAnyReg();
                movrc(reg2, param2);
                n := param1 * 8;
                Rex(0, reg2);
                OutByte2(01H, 45H + long(n) + (reg2 MOD 8) * 8);
                OutIntByte(n); (* add qword[rbp+n], reg2 *)
                drop
            ELSIF ABS(param2) = 1 THEN
                n := param1 * 8;
                OutByte3(48H, 0FFH, 45H + 8 * ORD(param2 = -1) + long(n)); (* inc/dec qword[rbp+n] *)
                OutIntByte(n)
            ELSE
                n := param1 * 8;
                OutByte3(48H, 81H + short(param2), 45H + long(n));
                OutIntByte(n);
                OutIntByte(param2) (* add qword[rbp+n], param2 *)
            END

        |IL.opLADR_INCCB, IL.opLADR_DECCB:
            param2 := param2 MOD 256;
            n := param1 * 8;
            OutByte2(80H, 45H + long(n) + 28H * ORD(opcode = IL.opLADR_DECCB));
            OutIntByte(n);
            OutByte(param2) (* add/sub byte[rbp+n], param2 *)

        |IL.opLADR_INC, IL.opLADR_DEC:
            UnOp(reg1);
            n := param2 * 8;
            Rex(0, reg1);
            OutByte2(01H + 28H * ORD(opcode = IL.opLADR_DEC), 45H + long(n) + (reg1 MOD 8) * 8);
            OutIntByte(n); (* add/sub qword[rbp+n], reg1 *)
            drop

        |IL.opLADR_INCB, IL.opLADR_DECB:
            UnOp(reg1);
            n := param2 * 8;
            IF reg1 >= 8 THEN
                OutByte(44H)
            END;
            OutByte2(28H * ORD(opcode = IL.opLADR_DECB), 45H + long(n) + 8 * (reg1 MOD 8));
            OutIntByte(n); (* add/sub byte[rbp+n], reg1_8 *)
            drop

        |IL.opLADR_INCL, IL.opLADR_EXCL:
            UnOp(reg1);
            cmprc(reg1, 64);
            n := param2 * 8;
            OutByte2(73H, 5 + 3 * ORD(~X86.isByte(n))); (* jnb L *)
            Rex(0, reg1);
            OutByte3(0FH, 0ABH + 8 * ORD(opcode = IL.opLADR_EXCL), 45H + long(n) + 8 * (reg1 MOD 8));
            OutIntByte(n); (* bts/btr qword[rbp+n], reg1 *)
            (* L: *)
            drop

        |IL.opLADR_INCLC, IL.opLADR_EXCLC:
            n := param1 * 8;
            OutByte3(48H, 0FH, 0BAH); (* bts/btr qword[rbp+n], param2 *)
            OutByte(6DH + long(n) + 8 * ORD(opcode = IL.opLADR_EXCLC));
            OutIntByte(n);
            OutByte(param2)

        |IL.opFNAME:
            fname := cmd(IL.FNAMECMD).fname

        END;

        cmd := cmd.next(COMMAND)
    END;

    ASSERT(R.pushed = 0);
    ASSERT(R.top = -1);
    ASSERT(xmm = -1)
END translate;


PROCEDURE prolog (modname: ARRAY OF CHAR; target, stack_size: INTEGER);
VAR
    ModName_Offs, entry, L: INTEGER;

BEGIN
    ModName_Offs := tcount * 8 + CHL.Length(IL.codes.data);
    Numbers_Offs := ModName_Offs + LENGTH(modname) + 1;
    ASSERT(UTILS.Align(Numbers_Offs, 16));

    entry := NewLabel();
    X86.SetLabel(entry);

    IF target = TARGETS.Win64DLL THEN
        dllret := NewLabel();
        push(r8);
        push(rdx);
        push(rcx);
        CallRTL(IL._dllentry);
        test(rax);
        jcc(je, dllret);
        pushc(0)
    ELSIF target = TARGETS.Linux64 THEN
        push(rsp)
    ELSE
        pushc(0)
    END;

    lea(rax, entry, sCODE);
    push(rax);
    pushDA(0); (* TYPES *)
    pushc(tcount);
    pushDA(ModName_Offs); (* MODNAME *)
    CallRTL(IL._init);

    IF target IN {TARGETS.Win64C, TARGETS.Win64GUI, TARGETS.Linux64} THEN
        L := NewLabel();
        pushc(0);
        push(rsp);
        pushc(1024 * 1024 * stack_size);
        pushc(0);
        CallRTL(IL._new);
        pop(rax);
        test(rax);
        jcc(je, L);
        GetRegA;
        addrc(rax, 1024 * 1024 * stack_size - 8);
        drop;
        mov(rsp, rax);
        X86.SetLabel(L)
    END
END prolog;


PROCEDURE epilog (modname: ARRAY OF CHAR; target: INTEGER);
VAR
    i, n: INTEGER;
    number: Number;
    exp: IL.EXPORT_PROC;


    PROCEDURE _import (imp: LISTS.LIST);
    VAR
        lib:  IL.IMPORT_LIB;
        proc: IL.IMPORT_PROC;

    BEGIN

        lib := imp.first(IL.IMPORT_LIB);
        WHILE lib # NIL DO
            BIN.Import(prog, lib.name, 0);
            proc := lib.procs.first(IL.IMPORT_PROC);
            WHILE proc # NIL DO
                BIN.Import(prog, proc.name, proc.label);
                proc := proc.next(IL.IMPORT_PROC)
            END;
            lib := lib.next(IL.IMPORT_LIB)
        END

    END _import;


BEGIN
    IF target = TARGETS.Win64DLL THEN
        X86.SetLabel(dllret);
        X86.ret
    ELSIF target = TARGETS.Linux64SO THEN
        sofinit := NewLabel();
        X86.ret;
        X86.SetLabel(sofinit);
        CallRTL(IL._sofinit);
        X86.ret
    ELSE
        pushc(0);
        CallRTL(IL._exit)
    END;

    X86.fixup;

    i := 0;
    WHILE i < tcount DO
        BIN.PutData64LE(prog, CHL.GetInt(IL.codes.types, i));
        INC(i)
    END;

    i := 0;
    WHILE i < CHL.Length(IL.codes.data) DO
        BIN.PutData(prog, CHL.GetByte(IL.codes.data, i));
        INC(i)
    END;

    BIN.PutDataStr(prog, modname);
    BIN.PutData(prog, 0);
    n := CHL.Length(prog.data);
    ASSERT(UTILS.Align(n, 16));
    i := n - CHL.Length(prog.data);
    WHILE i > 0 DO
        BIN.PutData(prog, 0);
        DEC(i)
    END;
    number := Numbers.first(Number);
    FOR i := 0 TO Numbers_Count - 1 DO
        BIN.PutData64LE(prog, number.value);
        number := number.next(Number)
    END;

    exp := IL.codes.export.first(IL.EXPORT_PROC);
    WHILE exp # NIL DO
        BIN.Export(prog, exp.name, exp.label);
        exp := exp.next(IL.EXPORT_PROC)
    END;

    _import(IL.codes._import)
END epilog;


PROCEDURE CodeGen* (outname: ARRAY OF CHAR; target: INTEGER; options: PROG.OPTIONS);
VAR
    path, modname, ext: PATHS.PATH;

BEGIN
    Xmm[0] := 0;
    tcount := CHL.Length(IL.codes.types);

    Win64RegPar[0] := rcx;
    Win64RegPar[1] := rdx;
    Win64RegPar[2] := r8;
    Win64RegPar[3] := r9;

    SystemVRegPar[0] := rdi;
    SystemVRegPar[1] := rsi;
    SystemVRegPar[2] := rdx;
    SystemVRegPar[3] := rcx;
    SystemVRegPar[4] := r8;
    SystemVRegPar[5] := r9;

    PATHS.split(outname, path, modname, ext);
    S.append(modname, ext);

    REG.Init(R, push, pop, mov, xchg, {rax, rcx, rdx, r8, r9, r10, r11});

    IL.set_bss(MAX(IL.codes.bss, MAX(IL.codes.dmin - CHL.Length(IL.codes.data), 8)));

    Numbers := LISTS.create(NIL);
    Numbers_Count := 0;
    NewNumber(ROR(1, 1));      (* 8000000000000000H *)
    NewNumber(0);
    NewNumber(ROR(-2, 1));     (* 7FFFFFFFFFFFFFFFH *)
    NewNumber(-1);
    NewNumber(ROR(7FFH, 12));  (* +Infinity *)
    NewNumber(ORD(-BITS(LSR(ASR(ROR(1, 1), 10), 1))));  (* {0..51, 63} *)
    NewNumber(LSR(ASR(ROR(1, 1), 9), 2));  (* {52..61} *)

    prog := BIN.create(IL.codes.lcount);
    BIN.SetParams(prog, IL.codes.bss, 1, WCHR(1), WCHR(0));

    X86.SetProgram(prog);

    prolog(modname, target, options.stack);
    translate(IL.codes.commands, tcount * 8);
    epilog(modname, target);

    BIN.fixup(prog);
    IF TARGETS.OS = TARGETS.osWIN64 THEN
        PE32.write(prog, outname, target = TARGETS.Win64C, target = TARGETS.Win64DLL, TRUE)
    ELSIF TARGETS.OS = TARGETS.osLINUX64 THEN
        ELF.write(prog, outname, sofinit, target = TARGETS.Linux64SO, TRUE)
    END
END CodeGen;


END AMD64.