;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;                                                                 ;;
;; Copyright (C) KolibriOS team 2004-2024. All rights reserved.    ;;
;; Distributed under terms of the GNU General Public License       ;;
;;                                                                 ;;
;;          GNU GENERAL PUBLIC LICENSE                             ;;
;;             Version 2, June 1991                                ;;
;;                                                                 ;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

format PE DLL native
entry START

API_VERSION = 0  ;debug
SRV_GETVERSION = 0
__DEBUG__ = 1
__DEBUG_LEVEL__ = 1
DRIVER_VERSION = 1
DBG_INFO = 1
NULLPTR = 0
FALSE = 0
TRUE = 1

section ".flat" code readable writable executable
include "../proc32.inc"
include "../struct.inc"
include "../macros.inc"
include "../fdo.inc"
include "../pci.inc"
include "../peimport.inc"
include "nvme.inc"
include "macros.inc"
include "lib.inc"
include "command.inc"

struct DISKMEDIAINFO
	flags 		dd ?
	sectorsize  	dd ?
	capacity 	dq ?
ends

proc START c, reason:dword

	cmp     [reason], DRV_ENTRY
	jne     .err

.entry:
	DEBUGF  DBG_INFO, "Detecting NVMe device...\n"
	call    detect_nvme
	test 	eax, eax
	jz      .err
	xor 	ebx, ebx
	mov 	esi, dword [p_nvme_devices]
	sub 	esi, sizeof.pcidev

.loop:
	add 	esi, sizeof.pcidev
	push 	ebx esi
	stdcall device_is_compat, esi
	test 	eax, eax
	jz 	.pop
	stdcall nvme_init, esi
	test 	eax, eax
	jz 	.pop
	pop 	esi ebx
	stdcall add_nvme_disk, esi
	jmp 	.next

.pop:
	pop 	esi ebx

.next:
	inc 	ebx
	cmp 	ebx, dword [pcidevs_len]
	jne 	.loop
	invoke  RegService, my_service, service_proc
	ret

.err:
	call    nvme_cleanup
      	xor     eax, eax
      	ret

endp

proc service_proc stdcall, ioctl:dword

      	mov     esi, [ioctl]
      	mov     eax, [esi + IOCTL.io_code]
      	cmp     eax, SRV_GETVERSION
      	jne     .ret

      	mov     eax, [esi + IOCTL.output]
      	cmp     [esi + IOCTL.out_size], 4
      	jne     .ret
      	mov     dword [eax], API_VERSION
      	xor     eax, eax
      	ret

.ret:
	or      eax, -1
	ret

endp

proc add_nvme_disk stdcall, pci:dword

	push 	esi
	mov 	esi, [pci]

	; NOTE: If the pcidev.num or pcidev.nsid is more than 9 then
	; this fails to build the string correctly. Ignoring this issue
	; for now since who has more than 9 NVMe SSDs on a desktop computer
	; and a NSID bigger than 9 is also unlikely.
	;
	; Still, will address this problem in the future.
	push 	0 ; null terminator
	movzx 	eax, byte [esi + pcidev.nsid]
	add 	al, "0"
	mov 	byte [esp], al
	dec 	esp
	mov 	byte [esp], "n"
	dec 	esp
	movzx 	eax, byte [esi + pcidev.num]
	add 	al, "0"
	mov 	byte [esp], al
	push 	"nvme"
	mov 	eax, esp
	invoke  DiskAdd, disk_functions, eax, [esi + pcidev.nsinfo], 0
	add 	esp, 10
	test 	eax, eax
	jz 	@f
	invoke  DiskMediaChanged, eax, 1
	DEBUGF  DBG_INFO, "nvme%un%u: Successfully registered disk\n", [esi + pcidev.num], [esi + pcidev.nsid]
	xor 	eax, eax
	inc 	eax
	pop 	esi
	ret

@@:
	DEBUGF  DBG_INFO, "nvme%un%u: Failed to register disk\n", [esi + pcidev.num], [esi + pcidev.nsid]
	xor 	eax, eax
	pop 	esi
	ret

endp

proc nvme_query_media stdcall, userdata:dword, info:dword

	push 	ebx esi edi
	mov 	esi, [userdata]
	mov 	ebx, dword [esi + NSINFO.pci]
	mov 	edi, [info]
	mov 	dword [edi + DISKMEDIAINFO.flags], 0
	mov 	cl, byte [esi + NSINFO.lbads]
	xor 	eax, eax
	inc 	eax
	shl 	eax, cl
	DEBUGF  DBG_INFO, "nvme%un%u (Query Media): Sector size = %u\n", [ebx + pcidev.num], [esi + NSINFO.nsid], eax
	mov 	dword [edi + DISKMEDIAINFO.sectorsize], eax
	mov 	eax, dword [esi + NSINFO.capacity]
	mov 	dword [edi + DISKMEDIAINFO.capacity], eax
	mov 	eax, dword [esi + NSINFO.capacity + 4]
	mov 	dword [edi + DISKMEDIAINFO.capacity + 4], eax
	DEBUGF  DBG_INFO, "nvme%un%u (Query Media): Capacity = %u + %u sectors\n", [ebx + pcidev.num], [esi + NSINFO.nsid], [esi + NSINFO.capacity], [esi + NSINFO.capacity + 4]
	xor 	eax, eax
	pop 	edi esi ebx
	ret

endp

; returns 1 if the given NSID is a an active NSID, returns
; 0 otherwise
proc is_active_namespace stdcall, pci:dword, nsid:dword
	
	push 	esi edi
	invoke  KernelAlloc, 0x1000
	test 	eax, eax
	jnz 	@f
	pop 	edi esi
	ret

@@:
	mov 	esi, eax
	invoke  GetPhysAddr
	stdcall nvme_identify, [pci], [nsid], eax, CNS_IDNS
	test 	eax, eax
	jz 	.not_active_nsid
	xor 	ecx, ecx

@@:
	mov 	eax, dword [esi + ecx * 4]
	test 	eax, eax
	jnz 	.is_active_nsid
	inc 	ecx
	cmp 	ecx, 0x1000 / 4
	jne 	@b
	
.not_active_nsid:
	invoke  KernelFree, esi
	pop 	edi esi
	xor 	eax, eax
	ret

.is_active_nsid:
	invoke  KernelFree, esi
	pop 	edi esi
	xor 	eax, eax
	inc 	eax
	ret

endp

; See page 248 of the NVMe 1.4 specification for reference
; Returns the number of namespaces that are active, note this
; doesn't mean if EAX = 5, then namespaces 1-5 will be active.
; This also sets [pci + pcidev.nn] and [pci + pcidev.nsids] 
; to appropriate values
proc determine_active_nsids stdcall, pci:dword

	push 	ebx esi
	mov 	esi, [pci]
	xor 	ebx, ebx
	xor 	ecx, ecx
	inc 	ecx

.loop:
	cmp 	ecx, dword [esi + pcidev.nn]
	ja 	.ret
	push 	ecx
	stdcall is_active_namespace, [pci], ecx
	pop 	ecx
	test 	eax, eax
	jz 	.not_active_namespace
	mov 	ebx, ecx
	jmp 	.ret

.not_active_namespace:
	inc 	ecx
	jmp 	.loop

.ret:
	pop 	edi esi
	mov 	eax, ebx
	ret

endp

proc build_prp_list stdcall, nprps:dword, buf:dword, prp_list_ptr:dword

	push 	esi ebx edi
	sub 	esp, 4

	; stack:
	; 	[esp]: virtual pointer to first PRP list
	; here, we store the pointer to the very first
	; PRP list so that free_prp_list can free the
	; entire PRP list if something goes wrong, it
	; also serves as our return value placeholder
	mov 	dword [esp], 0

	xor 	edi, edi
	xor 	esi, esi
	mov 	ecx, [nprps]
	shl 	ecx, 3 ; multiply by 8 since each PRP pointer is a QWORD

	; we'll store consecutive PRP list buffers here, for example
	; given 2 PRP lists, we allocate 2 continuous pages
	push 	ecx
	invoke  KernelAlloc, ecx ; store pointers to the PRP entries here
	pop 	ecx
	test 	eax, eax
	jz 	.err
	mov 	dword [esp], eax
	mov 	edi, eax
	mov 	eax, [prp_list_ptr]
	mov 	dword [eax], edi
	shr 	ecx, 1
	stdcall memsetdz, edi, ecx
	
	; note we assume buf is page-aligned
	mov 	esi, [buf]

.build_prp_list:
	; ensure we don't cross a page boundary
	mov 	ebx, [nprps]
	cmp 	ebx, PAGE_SIZE / 8
	jb 	@f
	mov 	ebx, PAGE_SIZE / 8
	sub 	[nprps], ebx

@@:
	xor 	ecx, ecx
	cmp 	dword [esp], edi
	je 	.loop

	; we need to store the pointer of the next
	; PRP list to the previous PRP list last entry
	mov 	eax, edi
	invoke  GetPhysAddr
	mov 	dword [edi - 8], eax
	mov 	dword [edi - 4], 0

.loop:
	mov 	eax, esi
	invoke  GetPhysAddr
	mov 	dword [edi + ecx * 8], eax
	mov 	dword [edi + ecx * 8 - 4], 0
	add 	esi, PAGE_SIZE
	inc 	ecx
	cmp 	ecx, ebx
	jne 	.loop

	; check if we we need to build another PRP list
	add 	edi, PAGE_SIZE
	cmp 	ebx, PAGE_SIZE / 8
	je 	.build_prp_list

	; PRP list successfully created
	mov 	eax, dword [esp]
	invoke  GetPhysAddr
	add 	esp, 4
	pop 	edi ebx esi
	ret

.err:
	add 	esp, 4
	pop 	edi ebx esi
	xor 	eax, eax
	ret
	
endp

proc alloc_dptr stdcall, ns:dword, prps_ptr:dword, numsectors:dword, prp_list_ptr:dword, buf:dword

	push 	ebx esi edi
	mov 	esi, [ns]
	mov 	edi, [prps_ptr]
	mov 	eax, [buf]
	invoke  GetPhysAddr
	mov 	dword [edi], eax
	mov 	cl, byte [esi + NSINFO.lbads]
	mov 	ebx, PAGE_SIZE
	shr 	ebx, cl
	mov 	edx, [numsectors]

	; is the buffer offset portion equal to 0?
	mov 	eax, [buf]
	mov 	ecx, eax
	and 	eax, PAGE_SIZE - 1
	jnz 	@f

	; is the number of sectors less than or equal to one memory page?
	cmp 	edx, ebx
	jbe 	.success
	shl 	ebx, 1 ; it is page aligned, so set ebx to 2 memory pages

@@:
	; is the number of sectors greater than one or two memory pages?
	cmp 	edx, ebx
	ja 	.build_prp_list
	
	; set PRP2
	mov 	eax, ecx
	and 	eax, not (PAGE_SIZE - 1)
	add 	eax, PAGE_SIZE
	invoke  GetPhysAddr
	mov 	dword [edi + 4], eax
	jmp 	.success

.build_prp_list:
	xchg 	ecx, ebx
	and 	ebx, not (PAGE_SIZE - 1)
	add 	ebx, PAGE_SIZE
	mov 	eax, [numsectors]
	mov 	ecx, ebx
	xor 	edx, edx
	div 	ecx
	stdcall build_prp_list, eax, ebx, [prp_list_ptr]
	test 	eax, eax
	jz 	.err
	mov 	dword [edi + 4], eax
	
.success:
	xor 	eax, eax
	inc 	eax
	pop 	edi esi ebx
	ret
	
.err:
	xor 	eax, eax
	pop 	edi esi ebx
	ret

endp

nvme_read:
	mov 	edx, NVM_CMD_READ
	jmp 	nvme_readwrite

nvme_write:
	mov 	edx, NVM_CMD_WRITE

proc nvme_readwrite stdcall, ns:dword, buf:dword, start_sector:qword, numsectors_ptr:dword

	push 	ebx esi edi
	sub 	esp, 20

	; TODO: check if numsectors exceeds IDENTC.MDTS?

	; stack:
	; [esp] - PRP1
	; [esp + 4] - PRP2
	; [esp + 8] - command type (read or write)
	; [esp + 12] - original numsectors value
	; [esp + 16] - virtual pointer to PRP2 PRP list (if allocated, 0 if not)
	mov 	ebx, esp

	mov 	esi, [ns]
	mov 	edi, [buf]

	mov 	eax, [numsectors_ptr]
	mov 	eax, dword [eax]
	DEBUGF  DBG_INFO, "buf: %x, start_sector: %u:%u, numsectors: %u\n", [buf], [start_sector + 4], [start_sector], eax
	mov 	dword [ebx + 4], 0 ; PRP2 entry (0 by default)
	mov 	dword [ebx + 8], edx ; command type (read or write)
	mov 	dword [ebx + 12], eax ; save original numsectors value
	mov 	dword [ebx + 16], 0 ; virtual pointer to PRP2 PRP list (not allocated by default)

	mov 	ecx, ebx
	add 	ecx, 16

	; Note that [esp] will contain the value of PRP1 and [esp + 4] will
	; contain the value of PRP2. If PRP2 is a PRP list, then [esp + 16] will point
	; to the allocated PRP list (after this call, only if it completes successfully)
	stdcall alloc_dptr, esi, ebx, eax, ecx, [buf]
	test 	eax, eax
	jz 	.fail

	DEBUGF  DBG_INFO, "PRP1: %x, PRP2: %x\n", [ebx], [ebx + 4]
	mov 	eax, dword [start_sector]

	; According to the NVMe specification, the NLB field in the I/O read and write
	; commands is a 0-based value (i.e., 0 is equivalant to 1, 1 is equivalant to 2, ...)
	; As far as I know, KolibriOS doesn't follow this mechanism so let's just decrement the
	; value and it should have the same effect.
	mov 	ecx, dword [ebx + 12]
	dec 	ecx

	; TODO: add non-blocking mechanisms later on
	push 	eax
	mov 	eax, dword [esi + NSINFO.pci]
	mov 	dword [eax + pcidev.spinlock], 1
	pop 	eax
	stdcall nvme_io_rw, [esi + NSINFO.pci], \
	                    1,  \
	                    [esi + NSINFO.nsid], \
	                    dword [ebx], \
			    dword [ebx + 4], \
	                    eax, \
	                    dword [start_sector + 4], \
	                    ecx, \
	                    dword [ebx + 8]

	; free PRP list (if allocated)
	mov 	edx, dword [ebx + 16]
	test 	edx, edx
	jz 	@f
	invoke  KernelFree, edx

@@:
	stdcall nvme_poll, [esi + NSINFO.pci]
	test 	eax, eax
	jz 	.fail
	xor 	eax, eax
	add 	esp, 20
	pop 	edi esi ebx
	ret

.fail:
	mov 	ebx, [numsectors_ptr]
	mov 	dword [ebx], 0
	or 	eax, -1 ; generic disk error

endp

proc detect_nvme

      	invoke  GetPCIList
	mov 	esi, eax
	mov 	ebx, eax

.check_dev:
	mov 	eax, dword [esi + PCIDEV.class]
	and	eax, 0x00ffff00 ; retrieve class/subclass code only
	cmp 	eax, 0x00010800 ; Mass Storage Controller - Non-Volatile Memory Controller
	je	.found_dev

.next_dev:
	mov	esi, dword [esi + PCIDEV.fd]
	cmp	esi, ebx
	jne 	.check_dev

.exit_success:
	xor 	eax, eax
	inc 	eax
	ret

.found_dev:
	cmp 	dword [pcidevs_len], TOTAL_PCIDEVS
	jne 	@f
	DEBUGF  DBG_INFO, "Can't add any more NVMe devices...\n"
	jmp 	.exit_success

@@:
	inc 	dword [pcidevs_len]
	cmp     dword [p_nvme_devices], 0
	jnz	@f ; was the pointer already allocated?
	invoke  KernelAlloc, sizeof.pcidev * TOTAL_PCIDEVS
	test 	eax, eax
	jz 	.err_no_mem
	mov     dword [p_nvme_devices], eax 

@@:
	mov 	ecx, dword [pcidevs_len]
	dec 	ecx
	mov 	edi, dword [p_nvme_devices]
	mov 	edx, ecx
	imul 	edx, sizeof.pcidev
	lea 	edi, [edi + edx]

	movzx 	eax, byte [esi + PCIDEV.bus]
	mov 	byte [edi + pcidev.bus], al
	movzx 	eax, byte [esi + PCIDEV.devfn]
	mov 	byte [edi + pcidev.devfn], al
	mov 	dword [edi + pcidev.num], ecx

	jmp	.next_dev

.err_no_mem:
	xor 	eax, eax
	ret

endp

proc device_is_compat stdcall, pci:dword

	push  	 esi edx ecx
	mov 	 esi, [pci]
	invoke 	 PciRead8, dword [esi + pcidev.bus], dword [esi + pcidev.devfn], PCI_header00.interrupt_line
	mov 	 byte [esi + pcidev.iline], al
	invoke   PciRead32, dword [esi + pcidev.bus], dword [esi + pcidev.devfn], PCI_header00.base_addr_0
	and      eax, 0xfffffff0
	test     eax, eax
	jz       .failure
	mov 	 edx, eax
	push 	 edx

	invoke   MapIoMem, eax, sizeof.NVME_MMIO, PG_SW+PG_NOCACHE
	test     eax, eax
	jz       .failure
	;DEBUGF   DBG_INFO, "(NVMe) MMIO allocated at: 0x%x\n", eax
	mov 	 dword [esi + pcidev.io_addr], eax
	mov 	 eax, dword [eax + NVME_MMIO.CAP + 4]
	and 	 eax, CAP_DSTRD
	mov 	 byte [esi + pcidev.dstrd], al
	
	; 1003h + ((2y + 1) * (4 << CAP.DSTRD))
	mov 	 eax, 4
	shl 	 ax, cl
	mov 	 ecx, NVM_ASQS
	shl 	 ecx, 1
	inc 	 ecx
	imul 	 ecx, eax
	add 	 ecx, 0x1003

	pop 	 edx
	invoke 	 MapIoMem, edx, ecx, PG_SW+PG_NOCACHE
	mov 	 dword [esi + pcidev.io_addr], eax
	mov 	 eax, dword [eax + NVME_MMIO.VS]
	DEBUGF   DBG_INFO, "nvme%u: Controller version: 0x%x\n", [esi + pcidev.num], eax
	mov 	 dword [esi + pcidev.version], eax
	pop 	 ecx edx esi
	xor 	 eax, eax
	inc 	 eax
	ret

.failure:
	PDEBUGF  DBG_INFO, "PCI(%u.%u.%u): something went wrong checking NVMe device compatibility\n", byte [esi + pcidev.bus], byte [esi + pcidev.devfn]
	pop 	 ecx edx esi
	xor      eax, eax
	ret
	
endp

; nvme_init: Initializes the NVMe controller 
proc nvme_init stdcall, pci:dword

	push 	 ebx esi edi
	mov 	 esi, dword [pci]

	; Check the PCI header to see if interrupts are disabled, if so
	; we have to re-enable them
	invoke   PciRead16, dword [esi + pcidev.bus], dword [esi + pcidev.devfn], PCI_header00.command
	and 	 eax, not (1 shl 10)
	invoke   PciWrite16, dword [esi + pcidev.bus], dword [esi + pcidev.devfn], PCI_header00.command, eax

	; Check if the device has a pointer to the capabilities list (status register bit 4 set to 1)
	; though this check is probably unnecessary since all PCIe devices should have this bit set to 1
	invoke   PciRead16, dword [esi + pcidev.bus], dword [esi + pcidev.devfn], PCI_header00.status
	test 	 ax, (1 shl 4)
	jz 	 .exit_fail

	invoke   PciRead8, dword [esi + pcidev.bus], dword [esi + pcidev.devfn], PCI_header00.cap_ptr
	and 	 eax, 0xfc ; bottom two bits are reserved, so mask them before we access the configuration space
	mov 	 edi, eax
	DEBUGF   DBG_INFO, "nvme%u: Checking capabilities...\n", [esi + pcidev.num]

; We need to check if there are any MSI/MSI-X capabilities, and if so, make sure they're disabled since
; we're using old fashioned pin-based interrupts (for now)
.read_cap:
	invoke   PciRead32, dword [esi + pcidev.bus], dword [esi + pcidev.devfn], edi
	add 	 edi, 2
	cmp 	 al, MSICAP_CID
	je 	 .got_msi_cap
	cmp 	 al, MSIXCAP_CID
	je 	 .got_msix_cap
	movzx 	 edi, ah
	test 	 edi, edi
	jnz 	 .read_cap
	DEBUGF   DBG_INFO, "nvme%u: MSI/MSI-X capability not found\n", [esi + pcidev.num]
	jmp 	 .end_cap_parse

.got_msi_cap:
	DEBUGF   DBG_INFO, "nvme%u: Found MSI capability\n", [esi + pcidev.num]
	invoke   PciRead32, dword [esi + pcidev.bus], dword [esi + pcidev.devfn], edi
	and      eax, not MSICAP_MSIE
	invoke   PciWrite32, dword [esi + pcidev.bus], dword [esi + pcidev.devfn], edi
	jmp 	 .end_cap_parse

.got_msix_cap:
	DEBUGF   DBG_INFO, "nvme%u: Found MSI-X capability\n", [esi + pcidev.num]
	invoke   PciRead32, dword [esi + pcidev.bus], dword [esi + pcidev.devfn], edi
	and 	 eax, not MSIXCAP_MXE
	invoke   PciWrite32, dword [esi + pcidev.bus], dword [esi + pcidev.devfn], edi

.end_cap_parse:
	mov      edi, dword [esi + pcidev.io_addr]
	if 0
		mov    eax, dword [edi + NVME_MMIO.CAP]
		DEBUGF DBG_INFO, "(NVMe) CAP (0-31): 0x%x\n", eax
		mov    eax, dword [edi + NVME_MMIO.CAP + 4]
		DEBUGF DBG_INFO, "(NVMe) CAP (32-63): 0x%x\n", eax
		mov    eax, dword [edi + NVME_MMIO.CC]
		DEBUGF DBG_INFO, "(NVMe) CC: 0x%x\n", eax
		mov    eax, dword [edi + NVME_MMIO.CSTS]
		DEBUGF DBG_INFO, "(NVMe) CSTS: 0x%x\n", eax
	end if

	; check maximum queue entries supported
	mov 	ax, word [edi + NVME_MMIO.CAP]
	cmp 	ax, SQ_ENTRIES
	jb 	.exit_fail
	
	; For some reason, bit 7 (No I/O command set supported) is also set to 1 despite bit 0 (NVM command set)
	; being set to 1.. so I am not sure if bit 7 should be checked at all.. investigate later.
	mov      eax, dword [edi + NVME_MMIO.CAP + 4]
	test 	 eax, CAP_CSS_NVM_CMDSET
	jz       .exit_fail
	DEBUGF   DBG_INFO, "nvme%u: OK... NVM command set supported\n", [esi + pcidev.num]

	; Reset controller before we configure it
	test 	 dword [edi + NVME_MMIO.CC], CC_EN
	jz 	 @f
	stdcall nvme_controller_reset, esi

@@:
	mov 	eax, dword [edi + NVME_MMIO.CAP + 4]
	and 	eax, CAP_MPSMIN
	shr 	eax, 16
	cmp 	eax, NVM_MPS
	ja 	.exit_fail
	mov 	eax, dword [edi + NVME_MMIO.CAP + 4]
	and 	eax, CAP_MPSMAX
	shr 	eax, 20
	cmp 	eax, NVM_MPS
	jb	.exit_fail
	DEBUGF  DBG_INFO, "nvme%u: OK... memory page size supported\n", [esi + pcidev.num]

	; Configure IOSQES, IOCQES, AMS, MPS, CSS
	and 	dword [edi + NVME_MMIO.CC], not (CC_AMS or CC_MPS or CC_CSS or CC_IOSQES or CC_IOCQES)
	mov 	eax, dword [edi + NVME_MMIO.CC]
	; 	CSS = 0 (NVM Command Set)
	; 	AMS = 0 (Round Robin)
	; 	MPS = 0 (4KiB Pages)
	; 	IOSQES = 6 (64B)
	; 	IOCQES = 4 (16B)
	or 	eax, (4 shl 20) or (6 shl 16)
	mov 	dword [edi + NVME_MMIO.CC], eax

	; Configure Admin Queue Attributes
	mov 	eax, dword [edi + NVME_MMIO.AQA]
	and 	eax, not (AQA_ASQS or AQA_ACQS)
	or 	eax, NVM_ASQS or (NVM_ACQS shl 16)
	mov 	dword [edi + NVME_MMIO.AQA], eax
	DEBUGF  DBG_INFO, "nvme%u: Admin queue attributes: 0x%x\n", [esi + pcidev.num], eax

	; Allocate list of queues
	invoke  KernelAlloc, sizeof.NVM_QUEUE_ENTRY * (LAST_QUEUE_ID + 1)
	test 	eax, eax
	jz 	.exit_fail
	mov 	dword [esi + pcidev.queue_entries], eax
	mov 	edi, eax
	stdcall memsetdz, eax, sizeof.NVM_QUEUE_ENTRY * (LAST_QUEUE_ID + 1) / 4

	; Allocate submission/completion queue pointers
	; TODO: Make these queues physically contiguous
	xor 	ecx, ecx

.init_queues:
	push 	ecx
	invoke  CreateRingBuffer, 0x1000, PG_SW
	pop 	ecx
	test 	eax, eax
	jz 	.exit_fail
	mov 	dword [edi + ecx + NVM_QUEUE_ENTRY.sq_ptr], eax
	push 	ecx
	stdcall memsetdz, eax, sizeof.SQ_ENTRY * SQ_ENTRIES / 4
	invoke  CreateRingBuffer, 0x1000, PG_SW
	pop 	ecx
	test 	eax, eax
	jz 	.exit_fail
	mov 	dword [edi + ecx + NVM_QUEUE_ENTRY.cq_ptr], eax
	push 	ecx
	stdcall memsetdz, eax, sizeof.CQ_ENTRY * CQ_ENTRIES / 4

	; Initialize command entries
	invoke  KernelAlloc, sizeof.NVMQCMD * CQ_ENTRIES
	pop 	ecx
	test 	eax, eax
	jz 	.exit_fail
	mov 	dword [edi + ecx + NVM_QUEUE_ENTRY.cmd_ptr], eax
	mov  	esi, eax
	push 	ecx
	xor 	ebx, ebx

.init_cmd_entries:
	; BUG: Won't pop ecx if allocation fails
	invoke  KernelAlloc, sizeof.MUTEX
	test 	eax, eax
	jz 	.exit_fail
	mov 	dword [esi + ebx * NVMQCMD.mutex_ptr], eax
	mov 	dword [esi + ebx * NVMQCMD.cid], ebx
	mov 	ecx, eax
	invoke  MutexInit
	inc 	ebx
	cmp 	ebx, CQ_ENTRIES
	jne 	.init_cmd_entries

	pop 	ecx
	add 	ecx, sizeof.NVM_QUEUE_ENTRY
	cmp 	ecx, (LAST_QUEUE_ID + 1) * sizeof.NVM_QUEUE_ENTRY
	jne 	.init_queues

	; Configure Admin Submission/Completion Queue Base Address
	mov 	esi, [pci]
	mov 	esi, dword [esi + pcidev.io_addr]
	mov 	eax, dword [edi + NVM_QUEUE_ENTRY.sq_ptr]
	invoke  GetPhysAddr
	push 	esi
	mov 	esi, [pci]
	DEBUGF  DBG_INFO, "nvme%u: Admin submission queue base address: 0x%x\n", [esi + pcidev.num], eax
	pop 	esi
	mov 	dword [esi + NVME_MMIO.ASQ], eax
	;mov 	dword [esi + NVME_MMIO.ASQ + 4], 0

	mov 	eax, dword [edi + NVM_QUEUE_ENTRY.cq_ptr]
	invoke  GetPhysAddr
	push 	esi
	mov 	esi, [pci]
	DEBUGF  DBG_INFO, "nvme%u: Admin completion queue base address: 0x%x\n", [esi + pcidev.num], eax
	pop 	esi
	mov 	dword [esi + NVME_MMIO.ACQ], eax
	;mov 	dword [esi + NVME_MMIO.ACQ + 4], 0

	; Attach interrupt handler
	mov 	esi, [pci]
	movzx 	eax, byte [esi + pcidev.iline]
	DEBUGF  DBG_INFO, "nvme%u: Attaching interrupt handler to IRQ %u\n", [esi + pcidev.num], eax
	invoke  AttachIntHandler, eax, irq_handler, 0
	test 	eax, eax
	jz 	.exit_fail
	DEBUGF  DBG_INFO, "nvme%u: Successfully attached interrupt handler\n", [esi + pcidev.num]

	; Restart the controller
	stdcall nvme_controller_start, esi

	invoke  KernelAlloc, 0x1000
	test 	eax, eax
	jz 	.exit_fail
	mov 	edi, eax
	invoke  GetPhysAddr
	;                      pci:dword, nsid:dword, dptr:dword, cns:byte
	stdcall nvme_identify, [pci],     0,          eax,        CNS_IDCS
	test 	eax, eax
	jz 	.exit_fail
	mov 	eax, dword [edi + IDENTC.nn]
	mov 	dword [esi + pcidev.nn], eax
	DEBUGF  DBG_INFO, "nvme%u: Namespace Count: %u\n", [esi + pcidev.num], eax
	lea 	ebx, byte [edi + IDENTC.sn]
	lea 	eax, byte [esi + pcidev.serial]
	stdcall memcpy, eax, ebx, 20
	DEBUGF  DBG_INFO, "nvme%u: Serial Number: %s\n", [esi + pcidev.num], eax
	add 	ebx, 20
	lea 	eax, byte [esi + pcidev.model]
	stdcall memcpy, eax, ebx, 40
	DEBUGF  DBG_INFO, "nvme%u: Model: %s\n", [esi + pcidev.num], eax
	mov 	edx, dword [esi + pcidev.version]
	
	cmp 	edx, VS140
	jb 	@f
	; This is a reserved field in pre-1.4 controllers
	mov 	al, byte [edi + IDENTC.cntrltype]
	cmp 	al, CNTRLTYPE_IO_CONTROLLER
	jne 	.exit_fail 	
	;DEBUGF  DBG_INFO, "nvme%u: I/O controller detected...\n", [esi + pcidev.num]

@@:
	; TODO: check IDENTC.AVSCC
	mov 	al, byte [edi + IDENTC.sqes]
	and 	al, 11110000b
	DEBUGF  DBG_INFO, "nvme%u: IDENTC.SQES = %u\n", [esi + pcidev.num], al
	cmp 	al, 0x60 ; maximum submission queue size should at least be 64 bytes
	jb 	.exit_fail
	mov 	al, byte [edi + IDENTC.cqes]
	and 	al, 11110000b
	DEBUGF  DBG_INFO, "nvme%u: IDENTC.CQES = %u\n", [esi + pcidev.num], al
	and 	al, 0x40 ; maximum completion queue entry size should at least be 16 bytes
	jb 	.exit_fail
	invoke  KernelFree, edi

	mov 	eax, 1 or (1 shl 16) ; CDW11 (set the number of queues we want)
	mov 	esi, [pci]
	mov 	dword [esi + pcidev.spinlock], 1
	stdcall set_features, [pci], NULLPTR, FID_NUMBER_OF_QUEUES, eax
	stdcall nvme_poll, esi
	test 	eax, eax
	jz 	.exit_fail
	mov 	esi, dword [esi + pcidev.queue_entries]
	mov 	esi, dword [esi + NVM_QUEUE_ENTRY.cq_ptr]
	mov 	eax, dword [esi + sizeof.CQ_ENTRY + CQ_ENTRY.cdw0]
	;DEBUGF  DBG_INFO, "nvme%u: Set Features CDW0: 0x%x\n", [esi + pcidev.num], eax
	test 	ax, ax ; Number of I/O Submission Queues allocated
	jz 	.exit_fail
	shl 	eax, 16
	test 	ax, ax ; Number of I/O Completion Queues allocated
	jnz	.exit_fail

	; Create I/O Queues
	; (TODO: create N queue pairs for N CPU cores, see page 8 of NVMe 1.4 spec for an explaination)
	mov 	esi, [pci]
	mov 	edi, esi
	mov 	esi, dword [esi + pcidev.queue_entries]
	lea 	esi, [esi + sizeof.NVM_QUEUE_ENTRY]
	mov 	eax, dword [esi + NVM_QUEUE_ENTRY.cq_ptr]
	invoke 	GetPhysAddr
	stdcall create_io_completion_queue, [pci], eax, 1, IEN_ON
	test 	eax, eax
	jz 	.exit_fail
	;DEBUGF  DBG_INFO, "nvme%u: Successfully created I/O completion queue 1\n", [edi + pcidev.num]
	mov 	eax, dword [esi + NVM_QUEUE_ENTRY.sq_ptr]
	invoke 	GetPhysAddr
	stdcall create_io_submission_queue, [pci], eax, 1, 1
	jz 	.exit_fail
	;DEBUGF  DBG_INFO, "nvme%u: Successfully created I/O submission queue 1\n", [edi + pcidev.num]

	stdcall determine_active_nsids, [pci]
	test 	eax, eax
	jz 	.exit_fail ; No active NSIDS
	mov 	esi, [pci]
	mov 	dword [esi + pcidev.nsid], eax
	DEBUGF  DBG_INFO, "nvme%u: Found active NSID: %u\n", [esi + pcidev.num], eax

	invoke  KernelAlloc, 0x1000
	test 	eax, eax
	jz 	.exit_fail
	mov 	edi, eax
	invoke  GetPhysAddr
	stdcall nvme_identify, [pci], [esi + pcidev.nsid], eax, CNS_IDNS
	test 	eax, eax
	jz 	.exit_fail
	invoke  KernelAlloc, sizeof.NSINFO
	test 	eax, eax
	jz 	.exit_fail
	mov 	ebx, eax
	mov 	dword [esi + pcidev.nsinfo], eax
	mov 	al, byte [edi + IDENTN.nsfeat]
	mov 	byte [ebx + NSINFO.features], al
	;DEBUGF  DBG_INFO, "nvme%un%u: Namespace Features: 0x%x\n", [esi + pcidev.num], [esi + pcidev.nsid], al
	mov 	eax, dword [esi + pcidev.nsid]
	mov 	dword [ebx + NSINFO.nsid], eax
	mov 	dword [ebx + NSINFO.pci], esi
	mov 	eax, dword [edi + IDENTN.nsze]
	mov 	dword [ebx + NSINFO.size], eax
	mov 	eax, dword [edi + IDENTN.nsze + 4]
	mov 	dword [ebx + NSINFO.size + 4], eax
	mov 	eax, dword [edi + IDENTN.ncap]
	mov 	dword [ebx + NSINFO.capacity], eax 
	mov 	eax, dword [edi + IDENTN.ncap + 4]
	mov 	dword [ebx + NSINFO.capacity + 4], eax 
	;DEBUGF  DBG_INFO, "nvme%un%u: Namespace Size: %u + %u logical blocks\n", [esi + pcidev.num], [esi + pcidev.nsid], [edi + IDENTN.nsze], [edi + IDENTN.nsze + 4]
	;DEBUGF  DBG_INFO, "nvme%un%u: Namespace Capacity: %u + %u logical blocks\n", [esi + pcidev.num], [esi + pcidev.nsid], [edi + IDENTN.ncap], [edi + IDENTN.ncap + 4]
	mov 	eax, dword [edi + IDENTN.lbaf0]
	shr 	eax, 16 ; Get LBADS

	; KolibriOS only supports a LBADS of 512, so if it's a higher value then we
	; have to ignore this namespace
	cmp 	al, SUPPORTED_LBADS
	jne 	.exit_fail

	mov 	byte [ebx + NSINFO.lbads], al
	invoke  KernelFree, edi
	if 0
		invoke  KernelAlloc, 0x6000
		test 	eax, eax
		jz 	.exit_fail
		mov 	edi, eax
		invoke  KernelAlloc, 0x8
		test 	eax, eax
		jz 	.exit_fail
		mov 	edx, NVM_CMD_READ
		mov 	dword [eax], 25
		add 	edi, 0x5
		mov 	dword [esi + pcidev.spinlock], 1
		stdcall nvme_readwrite, [esi + pcidev.nsinfo], edi, 0x1000, 0, eax
		stdcall nvme_poll, esi
		test 	eax, eax
		jz 	.exit_fail
	        DEBUGF  DBG_INFO, "STRING: %s\n", edi
		add 	edi, 0x2000
		DEBUGF  DBG_INFO, "STRING: %s\n", edi
	end if
	DEBUGF  DBG_INFO, "nvme%u: Successfully initialized driver\n", [esi + pcidev.num]
      	xor     eax, eax
      	inc     eax
	pop 	edi esi ebx
      	ret

.exit_fail:
	mov 	esi, [pci]
	DEBUGF  DBG_INFO, "nvme%u: Failed to initialize controller\n", [esi + pcidev.num]
	mov 	edi, dword [esi + pcidev.io_addr]
	mov 	eax, dword [edi + NVME_MMIO.CSTS]
	test 	eax, CSTS_CFS
	jz 	@f
	DEBUGF  DBG_INFO, "nvme%u: A fatal controller error has occurred\n", [esi + pcidev.num]

@@:
	xor 	eax, eax
	pop 	edi esi ebx
	ret

endp

proc get_new_cid stdcall, pci:dword, y:dword

	mov 	eax, [pci]
	mov 	eax, dword [eax + pcidev.queue_entries]
	mov 	ecx, [y]
	imul 	ecx, sizeof.NVM_QUEUE_ENTRY
	movzx 	eax, word [eax + ecx + NVM_QUEUE_ENTRY.head]
	ret

endp

proc nvme_controller_reset stdcall, pci:dword

	; TODO: Add timeout of CAP.TO seconds
	push 	esi edi
	mov 	esi, [pci]
	DEBUGF  DBG_INFO, "nvme%u: Resetting Controller...\n", [esi + pcidev.num]
	mov 	edi, dword [esi + pcidev.io_addr]
	and 	dword [edi + NVME_MMIO.CC], 0xfffffffe ; CC.EN = 0

; Wait for controller to be brought to idle state, CSTS.RDY should be cleared to 0 when this happens
.wait:
	test    dword [edi + NVME_MMIO.CSTS], CSTS_RDY
	jnz 	.wait
	DEBUGF  DBG_INFO, "nvme%u: Successfully reset controller...\n", [esi + pcidev.num]
	pop 	edi esi
	ret

endp

proc nvme_controller_start stdcall, pci:dword

	; TODO: Add timeout of CAP.TO seconds
	push 	esi edi
	mov 	esi, [pci]
	DEBUGF  DBG_INFO, "nvme%u: Starting Controller...\n", [esi + pcidev.num]
	mov 	edi, dword [esi + pcidev.io_addr]
	or 	dword [edi + NVME_MMIO.CC], 1 ; CC.EN = 1

; Wait for controller to be brought into active state, CSTS.RDY should be set to 1 when this happens
.wait:
	test 	dword [edi + NVME_MMIO.CSTS], CSTS_RDY
	jz 	.wait
	DEBUGF  DBG_INFO, "nvme%u: Successfully started controller...\n", [esi + pcidev.num]
	pop 	edi esi
	ret

endp

proc nvme_poll stdcall, pci:dword

	push 	esi
	mov 	esi, [pci]
	xor 	ecx, ecx

@@:
	inc 	ecx
	cmp 	ecx, 0xffffffff
	je 	@f
	xor 	eax, eax
	inc 	eax
	xchg 	eax, dword [esi + pcidev.spinlock]
	test 	eax, eax
	jnz 	@b

	; lock was released, return 1
	pop 	esi
	xor 	eax, eax
	inc 	eax
	ret

@@:
	; timeout: lock wasn't released, return 0
	pop 	esi
	xor 	eax, eax
	ret

endp


; Writes to completion queue 'y' head doorbell
proc cqyhdbl_write stdcall, pci:dword, y:dword, cqh:dword

	push 	esi edi
	mov 	esi, [pci]

	; 1000h + ((2y + 1) * (4 << CAP.DSTRD))
	mov 	eax, [y]
	shl 	al, 1
	inc 	al
	mov 	edx, 4
	mov 	cl, byte [esi + pcidev.dstrd]
	shl 	dx, cl
	imul 	dx, ax
	add 	dx, 0x1000
	mov 	ecx, [y]
	imul 	ecx, sizeof.NVM_QUEUE_ENTRY
	mov 	edi, dword [esi + pcidev.queue_entries]
	lea 	edi, dword [edi + ecx]
	mov 	esi, dword [esi + pcidev.io_addr]
	mov 	eax, [cqh]
	;DEBUGF  DBG_INFO, "Writing to completion queue doorbell register 0x%x: %u\n", edx, ax
	mov 	word [esi + edx], ax ; Write to CQyHDBL
	mov 	word [edi + NVM_QUEUE_ENTRY.head], ax
	pop 	edi esi
	ret

endp

; Writes to submission queue 'y' tail doorbell
proc sqytdbl_write stdcall, pci:dword, y:word, cmd:dword

	push 	ebx esi edi
	mov 	edi, [pci]
	mov 	edi, dword [edi + pcidev.queue_entries]
	movzx 	ecx, [y]
	imul 	ecx, sizeof.NVM_QUEUE_ENTRY
	;mov 	eax, dword [edi + ecx + NVM_QUEUE_ENTRY.cmd_ptr]
	mov 	edi, dword [edi + ecx + NVM_QUEUE_ENTRY.sq_ptr]
	mov 	esi, [cmd]
	mov 	ecx, dword [esi + SQ_ENTRY.cdw0]
	shr 	ecx, 16 ; Get CID
	imul 	ecx, sizeof.SQ_ENTRY
	lea 	edi, [edi + ecx]
	;mov 	ecx, [eax + ecx + NVMQCMD.mutex_ptr]
	;push 	ecx
	stdcall memcpy, edi, esi, sizeof.SQ_ENTRY
	;pop 	ecx
	;invoke  MutexLock

	mov 	edi, [pci]
	mov 	esi, dword [edi + pcidev.io_addr]
	mov 	edi, dword [edi + pcidev.queue_entries]
	movzx 	ecx, [y]
	imul 	ecx, sizeof.NVM_QUEUE_ENTRY
	movzx 	eax, word [edi + ecx + NVM_QUEUE_ENTRY.tail]
	cmp 	ax, (NVM_ASQS - 1)
	jb	@f 	
	xor 	ax, ax
	
@@:
	mov 	esi, [pci]
	inc 	ax
	; 1000h + (2y * (4 << CAP.DSTRD))
	movzx 	ebx, [y]
	shl 	ebx, 1
	mov 	edx, 4
	mov 	cl, byte [esi + pcidev.dstrd]
	shl 	edx, cl
	imul 	edx, ebx
	add 	edx, 0x1000
	;DEBUGF  DBG_INFO, "nvme%u: Writing to submission queue tail doorbell 0x%x: %u\n", [esi + pcidev.num], edx, ax
	mov 	esi, dword [esi + pcidev.io_addr]
	mov 	word [esi + edx], ax
	movzx 	ecx, [y]
	imul 	ecx, sizeof.NVM_QUEUE_ENTRY
	mov 	word [edi + ecx + NVM_QUEUE_ENTRY.tail], ax
	dec 	ax
	movzx 	ecx, [y]
	pop 	edi esi ebx
	ret

endp

proc is_queue_full stdcall, tail:word, head:word
	
	push 	bx
	mov 	ax, [tail]
	mov 	bx, [head]
	cmp 	ax, bx
	je 	.not_full
	test 	bx, bx
	jnz 	@f
	cmp 	ax, NVM_ASQS
	jne 	@f
	pop 	bx
	xor 	eax, eax
	inc 	eax
	ret

@@:
	cmp 	ax, bx
	jae 	.not_full
	sub 	ax, bx
	cmp 	ax, 1
	jne 	.not_full
	pop 	bx
	xor 	eax, eax
	inc 	eax
	ret
	
.not_full:
	pop 	bx
	xor 	eax, eax
	ret

endp

proc consume_cq_entries stdcall, pci:dword, queue:dword

	push 	esi edi
	mov 	esi, [pci]
	mov 	ecx, [queue]
	imul 	ecx, sizeof.NVM_QUEUE_ENTRY
	mov 	esi, dword [esi + pcidev.queue_entries]
	lea 	esi, [esi + ecx]
	mov 	edi, dword [esi + NVM_QUEUE_ENTRY.cq_ptr]
	movzx 	eax, word [esi + NVM_QUEUE_ENTRY.tail]
	movzx 	ecx, word [esi + NVM_QUEUE_ENTRY.head]
	;stdcall is_queue_full, eax, ecx
	;test 	eax, eax
	;jnz 	.end
	movzx 	ecx, word [esi + NVM_QUEUE_ENTRY.head]
	cmp 	ecx, (NVM_ACQS - 1)
	jb 	.loop
	xor 	ecx, ecx
	mov 	word [esi + NVM_QUEUE_ENTRY.head], cx
	
.loop:
	cmp 	cx, word [esi + NVM_QUEUE_ENTRY.tail]
	je 	.end
	mov 	edx, ecx
	imul 	edx, sizeof.CQ_ENTRY
	mov 	ax, word [edi + edx + CQ_ENTRY.status]
	;DEBUGF  DBG_INFO, "Status: 0x%x\n", ax
	inc 	cx
	push 	ecx
	stdcall cqyhdbl_write, [pci], [queue], ecx
	pop 	ecx
	jmp 	.loop

.end:
	pop 	edi esi
	xor 	eax, eax
	ret

endp

proc irq_handler

	push 	ebx esi edi
	mov 	edi, dword [p_nvme_devices]
	mov 	esi, edi
	sub 	esi, sizeof.pcidev
	mov 	ebx, dword [pcidevs_len]
	xor 	ecx, ecx

.check_who_raised_irq:
	add 	esi, sizeof.pcidev
	inc 	ecx
	cmp 	ecx, ebx
	ja 	.not_our_irq
	invoke 	PciRead16, dword [esi + pcidev.bus], dword [esi + pcidev.devfn], PCI_header00.status
	test 	al, 1000b ; check interrupt status
	jz 	.check_who_raised_irq
	
	mov 	edi, dword [esi + pcidev.io_addr]
	mov 	dword [edi + NVME_MMIO.INTMS], 0x3

	mov 	eax, dword [esi + pcidev.spinlock]
	test 	eax, eax
	jz 	@f ; not locked, so it must be an I/O command
	stdcall consume_cq_entries, esi, 0

@@:
	stdcall consume_cq_entries, esi, 1

	; Interrupt handled by driver, return 1
	mov 	dword [edi + NVME_MMIO.INTMC], 0x3
	xor 	eax, eax
	xchg 	eax, dword [esi + pcidev.spinlock] ; unlock spinlock
	pop 	edi esi ebx
	mov 	eax, 1
	ret

.not_our_irq:
	; Interrupt not handled by driver, return 0
	pop 	edi esi ebx
	xor 	eax, eax
	ret

endp

proc nvme_cleanup

	DEBUGF  DBG_INFO, "(NVMe): Cleaning up...\n"
	mov 	ecx, dword [pcidevs_len]
	mov 	eax, dword [p_nvme_devices]
	test 	eax, eax
	jnz 	.loop
	ret

.loop:
	;invoke  KernelFree, dword [p_nvme_devices + ecx * sizeof.pcidev + pcidev.ident_ptr]
	dec 	ecx
	test 	ecx, ecx
	jnz 	.loop
	invoke  KernelFree, dword [p_nvme_devices]

@@:
	ret

endp

;all initialized data place here
align 4
	p_nvme_devices dd 0
	pcidevs_len  dd 0
	my_service   db "NVMe",0  ;max 16 chars include zero
	disk_functions:
		dd 	 disk_functions.end - disk_functions
		dd 	 0 ; no close function
		dd 	 0 ; no closemedia function
		dd 	 nvme_query_media
		dd 	 nvme_read
		dd 	 nvme_write
		dd 	 0 ; no flush function
		dd 	 0 ; use default cache size
	.end:
	if __DEBUG__
	include_debug_strings
	end if

align 4
data fixups
end data

; vim: syntax=fasm