2024-04-17 21:44:10 +02:00
|
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
;; ;;
|
|
|
|
;; Copyright (C) KolibriOS team 2004-2024. All rights reserved. ;;
|
|
|
|
;; Distributed under terms of the GNU General Public License ;;
|
|
|
|
;; ;;
|
|
|
|
;; GNU GENERAL PUBLIC LICENSE ;;
|
|
|
|
;; Version 2, June 1991 ;;
|
|
|
|
;; ;;
|
|
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
2024-03-28 23:43:18 +01:00
|
|
|
|
|
|
|
format PE DLL native
|
|
|
|
entry START
|
|
|
|
|
|
|
|
API_VERSION equ 0 ;debug
|
|
|
|
SRV_GETVERSION equ 0
|
2024-03-30 04:42:01 +01:00
|
|
|
__DEBUG__ = 1
|
|
|
|
__DEBUG_LEVEL__ = 1
|
|
|
|
DRIVER_VERSION = 1
|
|
|
|
DBG_INFO = 1
|
|
|
|
|
2024-03-28 23:43:18 +01:00
|
|
|
section ".flat" code readable writable executable
|
|
|
|
include "../proc32.inc"
|
|
|
|
include "../struct.inc"
|
|
|
|
include "../macros.inc"
|
2024-03-30 04:42:01 +01:00
|
|
|
include "../fdo.inc"
|
2024-03-28 23:43:18 +01:00
|
|
|
include "../pci.inc"
|
2024-03-30 04:42:01 +01:00
|
|
|
include "../peimport.inc"
|
2024-03-31 21:43:38 +02:00
|
|
|
include "nvme.inc"
|
2024-04-21 02:29:06 +02:00
|
|
|
include "macros.inc"
|
2024-03-28 23:43:18 +01:00
|
|
|
|
|
|
|
proc START c, reason:dword
|
2024-04-17 21:35:01 +02:00
|
|
|
|
2024-05-26 19:24:34 +02:00
|
|
|
cmp [reason], DRV_ENTRY
|
|
|
|
jne .err
|
2024-04-17 21:35:01 +02:00
|
|
|
|
2024-03-28 23:43:18 +01:00
|
|
|
.entry:
|
2024-04-17 21:35:01 +02:00
|
|
|
DEBUGF DBG_INFO, "Detecting NVMe hardware...\n"
|
2024-05-26 19:24:34 +02:00
|
|
|
call detect_nvme
|
2024-04-28 02:11:57 +02:00
|
|
|
test eax, eax
|
2024-05-26 19:24:34 +02:00
|
|
|
jz .err
|
2024-04-20 05:39:34 +02:00
|
|
|
mov eax, dword [p_nvme_devices]
|
|
|
|
test eax, eax
|
2024-04-28 02:11:57 +02:00
|
|
|
jz .err
|
2024-04-28 19:50:40 +02:00
|
|
|
xor ecx, ecx
|
2024-04-17 21:35:01 +02:00
|
|
|
|
2024-04-28 19:50:40 +02:00
|
|
|
.loop:
|
2024-05-26 00:56:58 +02:00
|
|
|
mov ebx, dword [p_nvme_devices]
|
|
|
|
stdcall device_is_compat, ebx
|
2024-04-28 19:50:40 +02:00
|
|
|
test eax, eax
|
|
|
|
jz @f
|
2024-05-26 00:56:58 +02:00
|
|
|
stdcall nvme_init, ebx
|
2024-03-28 23:43:18 +01:00
|
|
|
|
2024-05-26 22:54:59 +02:00
|
|
|
;@@:
|
|
|
|
;inc ecx
|
|
|
|
;cmp ecx, dword [pcidevs_len]
|
|
|
|
;jne .loop
|
2024-05-26 19:24:34 +02:00
|
|
|
invoke RegService, my_service, service_proc
|
|
|
|
ret
|
2024-04-17 21:35:01 +02:00
|
|
|
|
|
|
|
.err:
|
|
|
|
call nvme_cleanup
|
2024-05-26 19:24:34 +02:00
|
|
|
xor eax, eax
|
|
|
|
ret
|
2024-04-17 21:35:01 +02:00
|
|
|
|
2024-03-28 23:43:18 +01:00
|
|
|
endp
|
|
|
|
|
|
|
|
proc service_proc stdcall, ioctl:dword
|
|
|
|
|
2024-05-26 19:24:34 +02:00
|
|
|
mov ebx, [ioctl]
|
|
|
|
mov eax, [ebx+IOCTL.io_code]
|
|
|
|
cmp eax, SRV_GETVERSION
|
|
|
|
jne @F
|
|
|
|
|
|
|
|
mov eax, [ebx+IOCTL.output]
|
|
|
|
cmp [ebx+IOCTL.out_size], 4
|
|
|
|
jne @F
|
|
|
|
mov dword [eax], API_VERSION
|
|
|
|
xor eax, eax
|
|
|
|
ret
|
2024-03-28 23:43:18 +01:00
|
|
|
|
|
|
|
@@:
|
|
|
|
or eax, -1
|
|
|
|
ret
|
2024-04-17 21:35:01 +02:00
|
|
|
|
2024-03-28 23:43:18 +01:00
|
|
|
endp
|
|
|
|
|
2024-03-31 21:43:38 +02:00
|
|
|
proc memset stdcall, p_data:dword, val:byte, sz:dword
|
2024-04-17 21:35:01 +02:00
|
|
|
|
2024-05-28 21:01:25 +02:00
|
|
|
push ebx ecx
|
|
|
|
mov eax, [p_data]
|
|
|
|
mov ecx, [sz]
|
2024-04-29 03:06:13 +02:00
|
|
|
mov bl, [val]
|
2024-05-26 19:24:34 +02:00
|
|
|
|
2024-03-31 21:43:38 +02:00
|
|
|
@@:
|
2024-05-28 21:01:25 +02:00
|
|
|
mov byte [eax + ecx], bl
|
|
|
|
dec ecx
|
|
|
|
test ecx, ecx
|
|
|
|
jnz @b
|
|
|
|
pop ecx ebx
|
2024-04-17 21:35:01 +02:00
|
|
|
ret
|
|
|
|
|
2024-03-31 21:43:38 +02:00
|
|
|
endp
|
|
|
|
|
2024-06-03 18:02:36 +02:00
|
|
|
; Submit a Administrator Command in the Submission Queue
|
|
|
|
proc submit_admin_cmd stdcall, sq_ptr:dword, slot:dword
|
2024-04-17 21:35:01 +02:00
|
|
|
|
2024-05-28 21:01:25 +02:00
|
|
|
mov esi, [sq_ptr]
|
2024-04-17 21:35:01 +02:00
|
|
|
xor eax, eax
|
|
|
|
ret
|
|
|
|
|
2024-03-31 21:43:38 +02:00
|
|
|
endp
|
|
|
|
|
2024-06-03 18:02:36 +02:00
|
|
|
proc nvme_identify stdcall, pci:dword, slot:dword, nsid:dword, dptr:dword, cid:word, cns:byte
|
2024-04-17 21:35:01 +02:00
|
|
|
|
2024-06-03 18:02:36 +02:00
|
|
|
push esi
|
|
|
|
mov esi, [pci]
|
|
|
|
mov esi, [esi + pcidev.sq_ptr]
|
|
|
|
mov eax, [slot]
|
|
|
|
imul eax, sizeof.SQ_ENTRY
|
|
|
|
add esi, eax
|
2024-05-28 21:01:25 +02:00
|
|
|
mov eax, [nsid]
|
2024-06-03 18:02:36 +02:00
|
|
|
mov dword [esi + SQ_ENTRY.nsid], eax
|
2024-05-28 21:01:25 +02:00
|
|
|
mov eax, [dptr]
|
2024-06-03 18:02:36 +02:00
|
|
|
mov dword [esi + SQ_ENTRY.dptr], eax
|
2024-05-28 21:02:24 +02:00
|
|
|
movzx eax, [cid]
|
2024-06-03 18:02:36 +02:00
|
|
|
or dword [esi + SQ_ENTRY.cdw0], ADM_CMD_IDENTIFY
|
|
|
|
or dword [esi + SQ_ENTRY.cdw0], eax
|
2024-05-28 21:01:25 +02:00
|
|
|
mov al, [cns]
|
2024-06-03 18:02:36 +02:00
|
|
|
mov byte [esi + SQ_ENTRY.cdw10], al
|
|
|
|
stdcall sqytdbl_write, [pci], 0, 0
|
|
|
|
pop esi
|
2024-04-17 21:35:01 +02:00
|
|
|
ret
|
|
|
|
|
2024-03-30 04:42:01 +01:00
|
|
|
endp
|
|
|
|
|
2024-04-17 21:35:01 +02:00
|
|
|
proc detect_nvme
|
|
|
|
|
2024-05-26 19:24:34 +02:00
|
|
|
invoke GetPCIList
|
2024-03-28 23:43:18 +01:00
|
|
|
mov edx, eax
|
2024-04-17 21:35:01 +02:00
|
|
|
|
2024-03-28 23:43:18 +01:00
|
|
|
.check_dev:
|
2024-04-17 21:35:01 +02:00
|
|
|
mov ebx, dword [eax + PCIDEV.class]
|
|
|
|
and ebx, 0x00ffff00 ; retrieve class/subclass code only
|
|
|
|
cmp ebx, 0x00010800 ; Mass Storage Controller - Non-Volatile Memory Controller
|
|
|
|
je .found_dev
|
|
|
|
|
2024-03-28 23:43:18 +01:00
|
|
|
.next_dev:
|
2024-04-17 21:35:01 +02:00
|
|
|
mov eax, dword [eax + PCIDEV.fd]
|
2024-03-28 23:43:18 +01:00
|
|
|
cmp eax, edx
|
|
|
|
jne .check_dev
|
2024-04-28 02:11:57 +02:00
|
|
|
jmp .exit_success
|
2024-04-17 21:35:01 +02:00
|
|
|
|
|
|
|
.found_dev:
|
2024-04-28 02:11:57 +02:00
|
|
|
push edx eax
|
2024-04-29 03:06:13 +02:00
|
|
|
PDEBUGF DBG_INFO, "PCI(%u.%u.%u): Detected NVMe device...\n", byte [eax + PCIDEV.bus], byte [eax + PCIDEV.devfn]
|
2024-04-28 02:11:57 +02:00
|
|
|
cmp dword [pcidevs_len], TOTAL_PCIDEVS
|
2024-04-20 05:39:34 +02:00
|
|
|
jne @f
|
2024-04-28 02:11:57 +02:00
|
|
|
pop eax edx
|
|
|
|
jmp .exit_success
|
2024-04-20 05:39:34 +02:00
|
|
|
|
|
|
|
@@:
|
2024-04-28 02:11:57 +02:00
|
|
|
inc dword [pcidevs_len]
|
2024-04-20 05:39:34 +02:00
|
|
|
mov ebx, dword [p_nvme_devices]
|
|
|
|
test ebx, ebx
|
|
|
|
jnz @f
|
2024-05-26 00:56:58 +02:00
|
|
|
invoke KernelAlloc, sizeof.pcidev
|
2024-04-20 05:39:34 +02:00
|
|
|
test eax, eax
|
2024-04-28 02:11:57 +02:00
|
|
|
jz .err_no_mem
|
|
|
|
mov dword [p_nvme_devices], eax
|
2024-05-26 00:56:58 +02:00
|
|
|
DEBUGF DBG_INFO, "(NVMe) Allocated pcidev struct at 0x%x\n", [p_nvme_devices]
|
2024-04-20 05:39:34 +02:00
|
|
|
|
|
|
|
@@:
|
|
|
|
mov ecx, dword [pcidevs_len]
|
2024-04-28 02:11:57 +02:00
|
|
|
dec ecx
|
2024-04-20 05:39:34 +02:00
|
|
|
pop eax
|
2024-05-26 00:56:58 +02:00
|
|
|
mov ebx, dword [p_nvme_devices]
|
|
|
|
|
2024-04-28 02:11:57 +02:00
|
|
|
movzx edx, byte [eax + PCIDEV.bus]
|
2024-05-26 00:56:58 +02:00
|
|
|
mov byte [ebx + pcidev.bus], dl
|
2024-04-28 02:11:57 +02:00
|
|
|
movzx edx, byte [eax + PCIDEV.devfn]
|
2024-05-26 00:56:58 +02:00
|
|
|
mov byte [ebx + pcidev.devfn], dl
|
|
|
|
|
2024-04-28 02:11:57 +02:00
|
|
|
pop edx
|
|
|
|
jmp .next_dev
|
|
|
|
|
|
|
|
.err_no_mem:
|
|
|
|
pop eax edx
|
|
|
|
xor eax, eax
|
|
|
|
ret
|
2024-04-20 05:39:34 +02:00
|
|
|
|
2024-04-28 02:11:57 +02:00
|
|
|
.exit_success:
|
|
|
|
xor eax, eax
|
|
|
|
inc eax
|
2024-04-17 21:35:01 +02:00
|
|
|
ret
|
|
|
|
|
|
|
|
endp
|
|
|
|
|
2024-04-29 03:06:13 +02:00
|
|
|
proc device_is_compat stdcall, pci:dword
|
2024-04-17 21:35:01 +02:00
|
|
|
|
2024-05-30 01:29:01 +02:00
|
|
|
push esi edx ecx
|
2024-05-29 21:43:14 +02:00
|
|
|
mov esi, [pci]
|
2024-05-30 22:06:50 +02:00
|
|
|
invoke PciRead8, dword [esi + pcidev.bus], dword [esi + pcidev.devfn], PCI_header00.interrupt_pin
|
|
|
|
test eax, eax
|
|
|
|
jz .failure
|
|
|
|
mov byte [esi + pcidev.ipin], al
|
|
|
|
invoke PciRead8, dword [esi + pcidev.bus], dword [esi + pcidev.devfn], PCI_header00.interrupt_line
|
|
|
|
mov byte [esi + pcidev.iline], al
|
2024-05-29 21:43:14 +02:00
|
|
|
invoke PciRead32, dword [esi + pcidev.bus], dword [esi + pcidev.devfn], PCI_header00.base_addr_0
|
2024-04-17 21:35:01 +02:00
|
|
|
and eax, 0xfffffff0
|
|
|
|
test eax, eax
|
2024-04-29 03:06:13 +02:00
|
|
|
jz .failure
|
2024-05-30 01:29:01 +02:00
|
|
|
mov edx, eax
|
|
|
|
push edx
|
2024-04-17 21:35:01 +02:00
|
|
|
|
2024-05-26 00:56:58 +02:00
|
|
|
invoke MapIoMem, eax, sizeof.NVME_MMIO, PG_SW+PG_NOCACHE
|
2024-04-17 21:35:01 +02:00
|
|
|
test eax, eax
|
2024-04-29 03:06:13 +02:00
|
|
|
jz .failure
|
2024-05-29 21:43:14 +02:00
|
|
|
;DEBUGF DBG_INFO, "(NVMe) MMIO allocated at: 0x%x\n", eax
|
|
|
|
mov dword [esi + pcidev.mmio_ptr], eax
|
|
|
|
mov eax, dword [eax + NVME_MMIO.CAP + 4]
|
|
|
|
and eax, CAP_DSTRD
|
|
|
|
|
|
|
|
; Stride is (2 ^ (2 + DSTRD)) bytes
|
|
|
|
add eax, 2
|
|
|
|
stdcall pow2, eax
|
2024-05-30 01:29:01 +02:00
|
|
|
mov ecx, eax
|
2024-05-29 21:43:14 +02:00
|
|
|
mov dword [esi + pcidev.dstrd], eax
|
|
|
|
|
2024-05-30 01:29:01 +02:00
|
|
|
; 1003h + ((2y + 1) * (4 << CAP.DSTRD))
|
|
|
|
mov eax, 4
|
|
|
|
shl ax, cl
|
|
|
|
mov ecx, NVM_ASQS
|
|
|
|
shl ecx, 1
|
|
|
|
inc ecx
|
|
|
|
imul ecx, eax
|
|
|
|
add ecx, 0x1003
|
|
|
|
|
|
|
|
pop edx
|
|
|
|
invoke MapIoMem, edx, ecx, PG_SW+PG_NOCACHE
|
|
|
|
mov dword [esi + pcidev.mmio_ptr], eax
|
|
|
|
mov eax, dword [eax + NVME_MMIO.VS]
|
2024-05-26 01:26:51 +02:00
|
|
|
DEBUGF DBG_INFO, "(NVMe) Controller version: 0x%x\n", eax
|
2024-05-30 01:29:01 +02:00
|
|
|
pop ecx edx esi
|
2024-04-28 02:11:57 +02:00
|
|
|
xor eax, eax
|
|
|
|
inc eax
|
|
|
|
ret
|
2024-04-20 05:39:34 +02:00
|
|
|
|
2024-04-29 03:06:13 +02:00
|
|
|
.failure:
|
2024-05-29 21:43:14 +02:00
|
|
|
PDEBUGF DBG_INFO, "PCI(%u.%u.%u): something went wrong checking NVMe device compatibility\n", byte [esi + pcidev.bus], byte [esi + pcidev.devfn]
|
2024-05-30 01:29:01 +02:00
|
|
|
pop ecx edx esi
|
2024-05-26 00:56:58 +02:00
|
|
|
xor eax, eax
|
2024-04-21 23:10:09 +02:00
|
|
|
ret
|
2024-04-20 05:39:34 +02:00
|
|
|
|
|
|
|
endp
|
|
|
|
|
|
|
|
; nvme_init: Initializes the NVMe controller
|
2024-04-29 03:06:13 +02:00
|
|
|
proc nvme_init stdcall, pci:dword
|
2024-04-20 05:39:34 +02:00
|
|
|
|
2024-05-30 22:06:50 +02:00
|
|
|
push ebx esi edi
|
2024-05-30 20:21:45 +02:00
|
|
|
mov esi, dword [pci]
|
|
|
|
mov edi, dword [esi + pcidev.mmio_ptr]
|
2024-05-26 01:26:51 +02:00
|
|
|
|
2024-05-29 21:43:14 +02:00
|
|
|
if 0
|
2024-05-26 22:54:59 +02:00
|
|
|
mov eax, dword [edi + NVME_MMIO.CAP]
|
|
|
|
DEBUGF DBG_INFO, "(NVMe) CAP (0-31): 0x%x\n", eax
|
|
|
|
mov eax, dword [edi + NVME_MMIO.CAP + 4]
|
|
|
|
DEBUGF DBG_INFO, "(NVMe) CAP (32-63): 0x%x\n", eax
|
|
|
|
mov eax, dword [edi + NVME_MMIO.CC]
|
|
|
|
DEBUGF DBG_INFO, "(NVMe) CC: 0x%x\n", eax
|
|
|
|
mov eax, dword [edi + NVME_MMIO.CSTS]
|
|
|
|
DEBUGF DBG_INFO, "(NVMe) CSTS: 0x%x\n", eax
|
2024-05-26 01:26:51 +02:00
|
|
|
end if
|
|
|
|
|
2024-05-26 22:54:59 +02:00
|
|
|
mov eax, dword [edi + NVME_MMIO.CAP]
|
|
|
|
test eax, CAP_CQR
|
2024-04-17 21:35:01 +02:00
|
|
|
jz .cqr_not_req
|
2024-03-28 23:43:18 +01:00
|
|
|
|
2024-04-02 01:47:14 +02:00
|
|
|
.cqr_not_req:
|
2024-05-24 01:15:34 +02:00
|
|
|
; For some reason, bit 7 (No I/O command set supported) is also set to 1 despite bit 0 (NVM command set)
|
|
|
|
; being set to 1.. so I am not sure if bit 7 should be checked at all.. investigate later.
|
2024-05-26 22:54:59 +02:00
|
|
|
mov eax, dword [edi + NVME_MMIO.CAP + 4]
|
|
|
|
test eax, CAP_CSS_NVM_CMDSET
|
2024-04-17 21:35:01 +02:00
|
|
|
jz .exit_fail
|
|
|
|
|
2024-05-26 02:37:42 +02:00
|
|
|
; Reset controller before we configure it
|
2024-05-26 23:19:25 +02:00
|
|
|
stdcall nvme_controller_reset, edi
|
2024-05-26 22:54:59 +02:00
|
|
|
mov eax, dword [edi + NVME_MMIO.CAP + 4]
|
|
|
|
and eax, CAP_MPSMIN
|
|
|
|
shr eax, 16
|
|
|
|
cmp eax, NVM_MPS
|
|
|
|
jg .exit_fail
|
|
|
|
mov eax, dword [edi + NVME_MMIO.CAP + 4]
|
|
|
|
and eax, CAP_MPSMAX
|
|
|
|
shr eax, 20
|
|
|
|
cmp eax, NVM_MPS
|
2024-05-26 02:37:42 +02:00
|
|
|
jl .exit_fail
|
|
|
|
|
2024-05-28 21:01:25 +02:00
|
|
|
; Configure AMS, MPS, CSS
|
2024-05-26 22:54:59 +02:00
|
|
|
mov eax, dword [edi + NVME_MMIO.CC]
|
2024-05-28 21:01:25 +02:00
|
|
|
and eax, not (CC_AMS or CC_MPS or CC_CSS)
|
|
|
|
or eax, 111b shl 4 ; Admin Command Set Only (temporary)
|
|
|
|
and dword [edi + NVME_MMIO.CC], eax
|
2024-05-26 02:37:42 +02:00
|
|
|
|
|
|
|
; Configure Admin Queue Attributes
|
2024-05-26 22:54:59 +02:00
|
|
|
mov eax, dword [edi + NVME_MMIO.AQA]
|
|
|
|
and eax, not (AQA_ASQS or AQA_ACQS)
|
2024-05-28 18:26:27 +02:00
|
|
|
or eax, NVM_ASQS or (NVM_ACQS shl 16)
|
2024-05-26 22:54:59 +02:00
|
|
|
mov dword [edi + NVME_MMIO.AQA], eax
|
2024-05-26 02:37:42 +02:00
|
|
|
|
|
|
|
; Configure Admin Submission/Completion Queue Base Address
|
2024-05-28 23:55:06 +02:00
|
|
|
invoke AllocPage
|
2024-05-26 22:54:59 +02:00
|
|
|
test eax, eax
|
|
|
|
jz .exit_fail
|
2024-05-30 22:06:50 +02:00
|
|
|
;or eax, PG_SW or PG_NOCACHE
|
2024-05-26 22:54:59 +02:00
|
|
|
mov dword [edi + NVME_MMIO.ASQ], eax
|
2024-05-26 23:19:25 +02:00
|
|
|
and dword [edi + NVME_MMIO.ASQ + 4], 0
|
2024-05-28 23:55:06 +02:00
|
|
|
invoke AllocPage
|
2024-05-26 22:54:59 +02:00
|
|
|
test eax, eax
|
|
|
|
jz .exit_fail
|
2024-05-30 22:06:50 +02:00
|
|
|
;or eax, PG_SW or PG_NOCACHE
|
2024-05-26 22:54:59 +02:00
|
|
|
mov dword [edi + NVME_MMIO.ACQ], eax
|
2024-05-26 23:19:25 +02:00
|
|
|
and dword [edi + NVME_MMIO.ACQ + 4], 0
|
2024-05-26 23:02:56 +02:00
|
|
|
|
|
|
|
; Restart the controller
|
2024-05-26 23:19:25 +02:00
|
|
|
stdcall nvme_controller_start, edi
|
2024-05-28 21:01:25 +02:00
|
|
|
|
2024-05-30 20:21:45 +02:00
|
|
|
invoke KernelAlloc, 0x2000
|
|
|
|
test eax, eax
|
|
|
|
jz .exit_fail
|
|
|
|
mov ebx, eax
|
|
|
|
invoke MapPage, ebx, dword [edi + NVME_MMIO.ASQ], PG_SW+PG_NOCACHE
|
|
|
|
mov dword [esi + pcidev.sq_ptr], eax
|
|
|
|
add ebx, 0x1000
|
|
|
|
invoke MapPage, ebx, dword [edi + NVME_MMIO.ACQ], PG_SW+PG_NOCACHE
|
|
|
|
mov dword [esi + pcidev.cq_ptr], eax
|
2024-05-30 22:06:50 +02:00
|
|
|
|
|
|
|
; Attach interrupt handler
|
|
|
|
movzx eax, byte [esi + pcidev.iline]
|
2024-05-30 23:11:24 +02:00
|
|
|
invoke AttachIntHandler, eax, irq_handler, esi
|
2024-05-30 22:06:50 +02:00
|
|
|
test eax, eax
|
|
|
|
jz .exit_fail
|
|
|
|
DEBUGF DBG_INFO, "(NVMe) Successfully attached interrupt handler\n"
|
|
|
|
|
2024-05-28 21:01:25 +02:00
|
|
|
invoke AllocPage
|
|
|
|
test eax, eax
|
|
|
|
jz .exit_fail
|
2024-06-03 18:02:36 +02:00
|
|
|
; pci:dword, nsid:dword, dptr:dword, cid:word, cns:byte
|
|
|
|
stdcall nvme_identify, [pci], 0, eax, 0, CNS_IDCS
|
2024-05-26 22:54:59 +02:00
|
|
|
|
2024-05-26 19:24:34 +02:00
|
|
|
xor eax, eax
|
|
|
|
inc eax
|
2024-05-30 22:06:50 +02:00
|
|
|
pop edi esi ebx
|
2024-05-26 19:24:34 +02:00
|
|
|
ret
|
2024-03-31 21:43:38 +02:00
|
|
|
|
2024-04-02 01:47:14 +02:00
|
|
|
.exit_fail:
|
2024-05-30 22:06:50 +02:00
|
|
|
DEBUGF DBG_INFO, "(NVMe) failed to initialize controller\n"
|
2024-04-20 05:39:34 +02:00
|
|
|
xor eax, eax
|
2024-05-30 22:06:50 +02:00
|
|
|
pop edi esi ebx
|
2024-03-31 21:43:38 +02:00
|
|
|
ret
|
2024-04-17 21:35:01 +02:00
|
|
|
|
2024-03-31 21:43:38 +02:00
|
|
|
endp
|
|
|
|
|
2024-05-26 23:19:25 +02:00
|
|
|
proc nvme_controller_reset stdcall, mmio:dword
|
2024-05-24 01:15:34 +02:00
|
|
|
|
2024-05-26 00:56:58 +02:00
|
|
|
DEBUGF DBG_INFO, "(NVMe) Resetting Controller...\n"
|
2024-05-26 23:19:25 +02:00
|
|
|
push edi
|
|
|
|
mov edi, dword [mmio]
|
|
|
|
and dword [edi + NVME_MMIO.CC], 0xfffffffe ; CC.EN = 0
|
|
|
|
stdcall nvme_wait, [mmio]
|
2024-05-24 01:15:34 +02:00
|
|
|
|
|
|
|
; Wait for controller to be brought to idle state, CSTS.RDY should be cleared to 0 when this happens
|
|
|
|
.wait:
|
2024-05-26 23:19:25 +02:00
|
|
|
test dword [edi + NVME_MMIO.CSTS], CSTS_RDY
|
2024-05-24 01:15:34 +02:00
|
|
|
jnz .wait
|
2024-05-26 00:56:58 +02:00
|
|
|
DEBUGF DBG_INFO, "(NVMe) Successfully reset controller...\n"
|
2024-05-26 23:19:25 +02:00
|
|
|
pop edi
|
2024-05-24 01:15:34 +02:00
|
|
|
ret
|
|
|
|
|
|
|
|
endp
|
|
|
|
|
2024-05-26 23:19:25 +02:00
|
|
|
proc nvme_controller_start stdcall, mmio:dword
|
2024-05-26 23:02:56 +02:00
|
|
|
|
|
|
|
DEBUGF DBG_INFO, "(NVMe) Starting Controller...\n"
|
2024-05-26 23:19:25 +02:00
|
|
|
push edi
|
|
|
|
mov edi, dword [mmio]
|
|
|
|
or dword [edi + NVME_MMIO.CC], 1 ;; CC.EN = 1
|
|
|
|
stdcall nvme_wait, [mmio]
|
2024-05-26 23:02:56 +02:00
|
|
|
|
|
|
|
.wait:
|
2024-05-26 23:19:25 +02:00
|
|
|
test dword [edi + NVME_MMIO.CSTS], CSTS_RDY
|
2024-05-26 23:02:56 +02:00
|
|
|
jz .wait
|
|
|
|
DEBUGF DBG_INFO, "(NVMe) Successfully started controller...\n"
|
2024-05-26 23:19:25 +02:00
|
|
|
pop edi
|
2024-05-26 23:02:56 +02:00
|
|
|
ret
|
|
|
|
|
|
|
|
endp
|
|
|
|
|
2024-05-26 00:56:58 +02:00
|
|
|
; Should be called only after the value of CC.EN has changed
|
2024-05-26 23:19:25 +02:00
|
|
|
proc nvme_wait stdcall, mmio:dword
|
|
|
|
|
2024-05-30 22:06:50 +02:00
|
|
|
push esi
|
2024-05-26 23:19:25 +02:00
|
|
|
mov esi, [mmio]
|
|
|
|
mov esi, dword [esi + NVME_MMIO.CAP]
|
|
|
|
and esi, CAP_TO
|
|
|
|
shr esi, 24
|
2024-05-26 00:56:58 +02:00
|
|
|
imul esi, 50
|
|
|
|
invoke Sleep
|
2024-05-30 22:06:50 +02:00
|
|
|
pop esi
|
2024-05-24 01:15:34 +02:00
|
|
|
ret
|
|
|
|
|
|
|
|
endp
|
|
|
|
|
2024-05-30 23:11:24 +02:00
|
|
|
; Writes to completion queue 'y' head doorbell
|
|
|
|
proc cqyhdbl_write stdcall, mmio:dword, dstrd:byte, y:byte
|
2024-05-28 23:55:06 +02:00
|
|
|
|
2024-05-30 23:11:24 +02:00
|
|
|
; 1000h + ((2y + 1) * (4 << CAP.DSTRD))
|
|
|
|
xor ecx, ecx
|
|
|
|
shl ecx, [y]
|
|
|
|
inc ecx
|
|
|
|
mov eax, 4
|
|
|
|
shl eax, [dstrd]
|
|
|
|
imul eax, ecx
|
2024-05-28 23:55:06 +02:00
|
|
|
add eax, 0x1000
|
2024-05-30 23:11:24 +02:00
|
|
|
ret
|
|
|
|
|
|
|
|
endp
|
2024-05-30 20:21:45 +02:00
|
|
|
|
2024-05-30 23:11:24 +02:00
|
|
|
; Writes to submission queue 'y' tail doorbell
|
2024-06-03 18:02:36 +02:00
|
|
|
proc sqytdbl_write stdcall, pci:dword, y:byte, offset:dword
|
|
|
|
|
|
|
|
push ebx esi edi
|
|
|
|
mov esi, [pci]
|
|
|
|
mov edi, [esi + pcidev.sq_ptr]
|
|
|
|
add edi, [offset]
|
|
|
|
mov esi, [esi + pcidev.mmio_ptr]
|
2024-05-30 23:11:24 +02:00
|
|
|
|
|
|
|
; 1000h + (2y * (4 << CAP.DSTRD))
|
2024-06-03 18:02:36 +02:00
|
|
|
mov ecx, dword [esi + pcidev.dstrd]
|
2024-05-30 23:11:24 +02:00
|
|
|
mov eax, 4
|
2024-06-03 18:02:36 +02:00
|
|
|
shl eax, cl
|
|
|
|
mov cl, [y]
|
|
|
|
xor ebx, ebx
|
|
|
|
shl ebx, cl
|
|
|
|
imul ebx, eax
|
|
|
|
add ebx, 0x1000
|
|
|
|
|
|
|
|
mov dword [esi + ebx], edi ; Write to register
|
|
|
|
pop edi esi ebx
|
2024-05-28 23:55:06 +02:00
|
|
|
ret
|
|
|
|
|
|
|
|
endp
|
|
|
|
|
|
|
|
; Calculates 2^x
|
2024-05-30 01:29:01 +02:00
|
|
|
proc pow2 stdcall, x:byte
|
2024-05-28 23:55:06 +02:00
|
|
|
|
|
|
|
push ecx
|
2024-05-30 01:29:01 +02:00
|
|
|
mov cl, [x]
|
2024-05-29 21:43:14 +02:00
|
|
|
xor eax, eax
|
|
|
|
inc eax
|
2024-05-30 01:29:01 +02:00
|
|
|
test cl, cl
|
2024-05-29 21:43:14 +02:00
|
|
|
jnz @f
|
2024-05-30 01:29:01 +02:00
|
|
|
pop ecx
|
2024-05-29 21:43:14 +02:00
|
|
|
ret
|
2024-05-29 21:57:45 +02:00
|
|
|
|
|
|
|
@@:
|
2024-05-30 01:29:01 +02:00
|
|
|
shl eax, cl
|
2024-05-28 23:55:06 +02:00
|
|
|
pop ecx
|
|
|
|
ret
|
|
|
|
|
|
|
|
endp
|
|
|
|
|
2024-05-30 22:06:50 +02:00
|
|
|
proc irq_handler
|
2024-06-03 18:02:36 +02:00
|
|
|
|
|
|
|
DEBUGF DBG_INFO, "HI, I GOT CALLED!\n"
|
|
|
|
|
2024-05-30 22:06:50 +02:00
|
|
|
endp
|
|
|
|
|
2024-03-31 21:43:38 +02:00
|
|
|
proc nvme_cleanup
|
2024-04-17 21:35:01 +02:00
|
|
|
|
2024-04-21 23:10:09 +02:00
|
|
|
DEBUGF DBG_INFO, "(NVMe): Cleaning up...\n"
|
2024-05-26 22:54:59 +02:00
|
|
|
mov ecx, dword [pcidevs_len]
|
2024-04-29 03:06:13 +02:00
|
|
|
mov eax, dword [p_nvme_devices]
|
|
|
|
test eax, eax
|
|
|
|
jnz .loop
|
2024-04-17 21:35:01 +02:00
|
|
|
ret
|
|
|
|
|
2024-04-29 03:06:13 +02:00
|
|
|
.loop:
|
|
|
|
;invoke KernelFree, dword [p_nvme_devices + ecx * sizeof.pcidev + pcidev.ident_ptr]
|
2024-05-26 00:56:58 +02:00
|
|
|
dec ecx
|
|
|
|
test ecx, ecx
|
|
|
|
jnz .loop
|
2024-04-29 03:06:13 +02:00
|
|
|
invoke KernelFree, dword [p_nvme_devices]
|
2024-05-26 22:54:59 +02:00
|
|
|
|
|
|
|
@@:
|
2024-04-29 03:06:13 +02:00
|
|
|
ret
|
2024-03-28 23:43:18 +01:00
|
|
|
|
2024-04-29 03:06:13 +02:00
|
|
|
endp
|
2024-03-28 23:43:18 +01:00
|
|
|
|
|
|
|
;all initialized data place here
|
|
|
|
align 4
|
2024-04-20 05:39:34 +02:00
|
|
|
p_nvme_devices dd 0
|
|
|
|
pcidevs_len dd 0
|
2024-04-17 22:04:49 +02:00
|
|
|
my_service db "NVMe",0 ;max 16 chars include zero
|
2024-05-04 00:07:31 +02:00
|
|
|
if __DEBUG__
|
2024-04-17 21:35:01 +02:00
|
|
|
include_debug_strings
|
2024-05-04 00:07:31 +02:00
|
|
|
end if
|
2024-03-28 23:43:18 +01:00
|
|
|
|
|
|
|
align 4
|
|
|
|
data fixups
|
|
|
|
end data
|