summaryrefslogtreecommitdiffstats
path: root/dma
diff options
context:
space:
mode:
Diffstat (limited to 'dma')
-rw-r--r--dma/ipe.c100
-rw-r--r--dma/ipe.h3
-rw-r--r--dma/ipe_benchmark.c2
-rw-r--r--dma/ipe_private.h12
4 files changed, 57 insertions, 60 deletions
diff --git a/dma/ipe.c b/dma/ipe.c
index 953de09..3e46cb3 100644
--- a/dma/ipe.c
+++ b/dma/ipe.c
@@ -22,8 +22,8 @@
pcilib_dma_context_t *dma_ipe_init(pcilib_t *pcilib, const char *model, const void *arg) {
-// int err = 0;
-
+ pcilib_register_value_t value;
+
const pcilib_model_description_t *model_info = pcilib_get_model_description(pcilib);
ipe_dma_t *ctx = malloc(sizeof(ipe_dma_t));
@@ -32,11 +32,6 @@ pcilib_dma_context_t *dma_ipe_init(pcilib_t *pcilib, const char *model, const vo
memset(ctx, 0, sizeof(ipe_dma_t));
ctx->dmactx.pcilib = pcilib;
-#ifdef IPEDMA_64BIT_MODE
- // Always supported and we need to use it
- ctx->mode64 = 1;
-#endif /* IPEDMA_64BIT_MODE */
-
pcilib_register_bank_t dma_bank = pcilib_find_register_bank_by_addr(pcilib, PCILIB_REGISTER_BANK_DMA);
if (dma_bank == PCILIB_REGISTER_BANK_INVALID) {
@@ -47,6 +42,19 @@ pcilib_dma_context_t *dma_ipe_init(pcilib_t *pcilib, const char *model, const vo
ctx->dma_bank = model_info->banks + dma_bank;
ctx->base_addr = pcilib_resolve_register_address(pcilib, ctx->dma_bank->bar, ctx->dma_bank->read_addr);
+
+ RD(IPEDMA_REG_PCIE_GEN, value);
+
+#ifdef IPEDMA_ENFORCE_64BIT_MODE
+ ctx->mode64 = 1;
+#else /* IPEDMA_ENFORCE_64BIT_MODE */
+ // According to Lorenzo, some gen2 boards have problems with 64-bit addressing. Therefore, we only enable it for gen3 boards unless enforced
+ if ((value&IPEDMA_MASK_PCIE_GEN) > 2) ctx->mode64 = 1;
+#endif /* IPEDMA_ENFORCE_64BIT_MODE */
+
+#ifdef IPEDMA_STREAMING_MODE
+ if (value&IPEDMA_MASK_STREAMING_MODE) ctx->streaming = 1;
+#endif /* IPEDMA_STREAMING_MODE */
}
return (pcilib_dma_context_t*)ctx;
@@ -63,7 +71,7 @@ void dma_ipe_free(pcilib_dma_context_t *vctx) {
int dma_ipe_start(pcilib_dma_context_t *vctx, pcilib_dma_engine_t dma, pcilib_dma_flags_t flags) {
- size_t i;
+ size_t i, num_pages;
ipe_dma_t *ctx = (ipe_dma_t*)vctx;
@@ -110,15 +118,16 @@ int dma_ipe_start(pcilib_dma_context_t *vctx, pcilib_dma_engine_t dma, pcilib_dm
if ((reuse_desc & PCILIB_KMEM_REUSE_PERSISTENT) == 0) pcilib_warning("Lost DMA buffers are found (non-persistent mode), reinitializing...");
else if ((reuse_desc & PCILIB_KMEM_REUSE_HARDWARE) == 0) pcilib_warning("Lost DMA buffers are found (missing HW reference), reinitializing...");
else {
-#ifndef IPEDMA_BUG_DMARD
-# ifndef IPEDMA_STREAMING_MODE
- RD(IPEDMA_REG_PAGE_COUNT, value);
-
- if (value != IPEDMA_DMA_PAGES) pcilib_warning("Inconsistent DMA buffers are found (Number of allocated buffers (%lu) does not match current request (%lu)), reinitializing...", value + 1, IPEDMA_DMA_PAGES);
- else
-# endif /* IPEDMA_STREAMING_MODE */
-#endif /* IPEDMA_BUG_DMARD */
+ if (ctx->streaming)
preserve = 1;
+ else {
+ RD(IPEDMA_REG_PAGE_COUNT, value);
+
+ if (value != IPEDMA_DMA_PAGES)
+ pcilib_warning("Inconsistent DMA buffers are found (Number of allocated buffers (%lu) does not match current request (%lu)), reinitializing...", value + 1, IPEDMA_DMA_PAGES);
+ else
+ preserve = 1;
+ }
}
}
} else pcilib_warning("Inconsistent DMA buffers (modes of ring and page buffers does not match), reinitializing....");
@@ -132,12 +141,6 @@ int dma_ipe_start(pcilib_dma_context_t *vctx, pcilib_dma_engine_t dma, pcilib_dm
ctx->preserve = 1;
// Detect the current state of DMA engine
-#ifdef IPEDMA_BUG_DMARD
- FILE *f = fopen("/tmp/pcitool_lastread", "r");
- if (!f) pcilib_error("Can't read current status");
- fread(&value, 1, sizeof(pcilib_register_value_t), f);
- fclose(f);
-#else /* IPEDMA_BUG_DMARD */
RD(IPEDMA_REG_LAST_READ, value);
// Numbered from 1 in FPGA
# ifdef IPEDMA_BUG_LAST_READ
@@ -146,7 +149,6 @@ int dma_ipe_start(pcilib_dma_context_t *vctx, pcilib_dma_engine_t dma, pcilib_dm
# else /* IPEDMA_BUG_LAST_READ */
value--;
# endif /* IPEDMA_BUG_LAST_READ */
-#endif /* IPEDMA_BUG_DMARD */
ctx->last_read = value;
} else {
@@ -162,12 +164,10 @@ int dma_ipe_start(pcilib_dma_context_t *vctx, pcilib_dma_engine_t dma, pcilib_dm
WR(IPEDMA_REG_RESET, 0x0);
usleep(IPEDMA_RESET_DELAY);
-#ifndef IPEDMA_BUG_DMARD
// Verify PCIe link status
RD(IPEDMA_REG_RESET, value);
if ((value != 0x14031700)&&(value != 0x14021700))
pcilib_warning("PCIe is not ready, code is %lx", value);
-#endif /* IPEDMA_BUG_DMARD */
// Enable 64 bit addressing and configure TLP and PACKET sizes (40 bit mode can be used with big pre-allocated buffers later)
if (ctx->mode64) address64 = 0x8000 | (0<<24);
@@ -203,7 +203,13 @@ int dma_ipe_start(pcilib_dma_context_t *vctx, pcilib_dma_engine_t dma, pcilib_dm
// Instructing DMA engine that writting should start from the first DMA page
*last_written_addr_ptr = 0;
- for (i = 0; i < IPEDMA_DMA_PAGES; i++) {
+ // In ring buffer mode, the hardware taking care to preserve an empty buffer to help distinguish between
+ // completely empty and completely full cases. In streaming mode, it is our responsibility to track this
+ // information. Therefore, we always keep the last buffer free
+ num_pages = IPEDMA_DMA_PAGES;
+ if (ctx->streaming) num_pages--;
+
+ for (i = 0; i < num_pages; i++) {
uintptr_t bus_addr_check, bus_addr = pcilib_kmem_get_block_ba(ctx->dmactx.pcilib, pages, i);
WR(IPEDMA_REG_PAGE_ADDR, bus_addr);
if (bus_addr%4096) printf("Bad address %lu: %lx\n", i, bus_addr);
@@ -220,14 +226,6 @@ int dma_ipe_start(pcilib_dma_context_t *vctx, pcilib_dma_engine_t dma, pcilib_dm
WR(IPEDMA_REG_CONTROL, 0x1);
ctx->last_read = IPEDMA_DMA_PAGES - 1;
-
-#ifdef IPEDMA_BUG_DMARD
- FILE *f = fopen("/tmp/pcitool_lastread", "w");
- if (!f) pcilib_error("Can't write current status");
- value = ctx->last_read;
- fwrite(&value, 1, sizeof(pcilib_register_value_t), f);
- fclose(f);
-#endif /* IPEDMA_BUG_DMARD */
}
ctx->last_read_addr = pcilib_kmem_get_block_ba(ctx->dmactx.pcilib, pages, ctx->last_read);
@@ -411,10 +409,6 @@ int dma_ipe_stream_read(pcilib_dma_context_t *vctx, pcilib_dma_engine_t dma, uin
pcilib_dma_flags_t packet_flags = PCILIB_DMA_FLAG_EOP;
size_t nodata_sleep;
-#ifdef IPEDMA_BUG_DMARD
- pcilib_register_value_t value;
-#endif /* IPEDMA_BUG_DMARD */
-
switch (sched_getscheduler(0)) {
case SCHED_FIFO:
case SCHED_RR:
@@ -507,16 +501,21 @@ int dma_ipe_stream_read(pcilib_dma_context_t *vctx, pcilib_dma_engine_t dma, uin
// pcilib_kmem_sync_block(ctx->dmactx.pcilib, ctx->pages, PCILIB_KMEM_SYNC_TODEVICE, cur_read);
// Return buffer into the DMA pool when processed
-#ifdef IPEDMA_STREAMING_MODE
- uintptr_t buf_ba = pcilib_kmem_get_block_ba(ctx->dmactx.pcilib, ctx->pages, cur_read);
- WR(IPEDMA_REG_PAGE_ADDR, buf_ba);
+ if (ctx->streaming) {
+ size_t last_free;
+ // We always keep 1 buffer free to distinguish between completely full and empty cases
+ if (cur_read) last_free = cur_read - 1;
+ else last_free = IPEDMA_DMA_PAGES - 1;
+
+ uintptr_t buf_ba = pcilib_kmem_get_block_ba(ctx->dmactx.pcilib, ctx->pages, last_free);
+ WR(IPEDMA_REG_PAGE_ADDR, buf_ba);
# ifdef IPEDMA_STREAMING_CHECKS
- pcilib_register_value_t streaming_status;
- RD(IPEDMA_REG_STREAMING_STATUS, streaming_status);
- if (streaming_status)
- pcilib_error("Invalid status (0x%lx) adding a DMA buffer into the queue", streaming_status);
+ pcilib_register_value_t streaming_status;
+ RD(IPEDMA_REG_STREAMING_STATUS, streaming_status);
+ if (streaming_status)
+ pcilib_error("Invalid status (0x%lx) adding a DMA buffer into the queue", streaming_status);
# endif /* IPEDMA_STREAMING_MODE */
-#endif /* IPEDMA_STREAMING_MODE */
+ }
// Numbered from 1
#ifdef IPEDMA_BUG_LAST_READ
@@ -533,15 +532,6 @@ int dma_ipe_stream_read(pcilib_dma_context_t *vctx, pcilib_dma_engine_t dma, uin
ctx->last_read = cur_read;
ctx->last_read_addr = pcilib_kmem_get_block_ba(ctx->dmactx.pcilib, ctx->pages, cur_read);
-
-#ifdef IPEDMA_BUG_DMARD
- FILE *f = fopen("/tmp/pcitool_lastread", "w");
- if (!f) pcilib_error("Can't write current status");
- value = cur_read;
- fwrite(&value, 1, sizeof(pcilib_register_value_t), f);
- fclose(f);
-#endif /* IPEDMA_BUG_DMARD */
-
} while (ret);
return 0;
diff --git a/dma/ipe.h b/dma/ipe.h
index 5640606..7f32d61 100644
--- a/dma/ipe.h
+++ b/dma/ipe.h
@@ -65,6 +65,9 @@ static const pcilib_register_description_t ipe_dma_registers[] = {
{0x000C, 24, 8, 0, 0xFFFFFFFF, PCILIB_REGISTER_RW , PCILIB_REGISTER_BITS, PCILIB_REGISTER_BANK_DMA, "mwr_up_addr", "Upper address for 64 bit memory addressing"},
{0x0010, 0, 32, 0, 0x00000000, PCILIB_REGISTER_RW , PCILIB_REGISTER_STANDARD, PCILIB_REGISTER_BANK_DMA, "mwr_count", "Write DMA TLP Count"},
{0x0014, 0, 32, 0, 0x00000000, PCILIB_REGISTER_RW , PCILIB_REGISTER_STANDARD, PCILIB_REGISTER_BANK_DMA, "mwr_pattern", "DMA generator data pattern"},
+ {0x0018, 0, 32, 0, 0x00000000, PCILIB_REGISTER_R , PCILIB_REGISTER_STANDARD, PCILIB_REGISTER_BANK_DMA, "dma_mode_flags", "DMA operation mode"},
+ {0x0018, 0, 4, 0, 0x00000000, PCILIB_REGISTER_R , PCILIB_REGISTER_BITS, PCILIB_REGISTER_BANK_DMA, "pcie_gen", "PCIe version 2/3 depending on the used XILINX core"},
+ {0x0018, 4, 1, 0, 0x00000000, PCILIB_REGISTER_R , PCILIB_REGISTER_BITS, PCILIB_REGISTER_BANK_DMA, "streaming_dma", "Streaming mode (enabled/disabled)"},
{0x0028, 0, 32, 0, 0x00000000, PCILIB_REGISTER_R , PCILIB_REGISTER_STANDARD, PCILIB_REGISTER_BANK_DMA, "mwr_perf", "MWR Performance"},
{0x003C, 0, 32, 0, 0x00000000, PCILIB_REGISTER_R , PCILIB_REGISTER_STANDARD, PCILIB_REGISTER_BANK_DMA, "cfg_lnk_width", "Negotiated and max width of PCIe Link"},
{0x003C, 0, 6, 0, 0x00000000, PCILIB_REGISTER_R , PCILIB_REGISTER_BITS, PCILIB_REGISTER_BANK_DMA, "cfg_cap_max_lnk_width", "Max link width"},
diff --git a/dma/ipe_benchmark.c b/dma/ipe_benchmark.c
index f8e0a7e..3ce04d3 100644
--- a/dma/ipe_benchmark.c
+++ b/dma/ipe_benchmark.c
@@ -166,7 +166,7 @@ double dma_ipe_benchmark(pcilib_dma_context_t *vctx, pcilib_dma_engine_addr_t dm
for (bytes = 0; bytes < size; bytes += rbytes) {
err = read_dma(ctx->dmactx.pcilib, 0, addr, size - bytes, PCILIB_DMA_FLAG_MULTIPACKET, PCILIB_DMA_TIMEOUT, buf + bytes, &rbytes);
if (err) {
- pcilib_error("Can't read data from DMA, error %i", err);
+ pcilib_error("Can't read data from DMA (iteration: %zu, offset: %zu), error %i", iter, bytes, err);
return -1;
}
}
diff --git a/dma/ipe_private.h b/dma/ipe_private.h
index 249286d..5054a58 100644
--- a/dma/ipe_private.h
+++ b/dma/ipe_private.h
@@ -3,13 +3,13 @@
#include "dma.h"
-#define IPEDMA_64BIT_MODE 1 /**< 64-bit mode addressing is required to support PCIe gen3 */
+//#define IPEDMA_ENFORCE_64BIT_MODE 1 /**< enforce 64-bit mode addressing (otherwise it is used only if register 0x18 specifies PCIe gen3 as required by DMA engine) */
#define IPEDMA_CORES 1
#define IPEDMA_MAX_TLP_SIZE 256 /**< Defines maximum TLP in bytes supported by device */
-//#define IPEDMA_TLP_SIZE 128 /**< If set, enforces the specified TLP size */
+//#define IPEDMA_TLP_SIZE 128 /**< If set, enforces the specified TLP size */
#define IPEDMA_STREAMING_MODE /**< Enables streaming DMA operation mode instead of ring-buffer, the page is written once and forgotten and need to be pushed in queue again */
-#define IPEDMA_STREAMING_CHECKS /**< Enables status checks in streaming mode (it will cause performance penalty) */
+//#define IPEDMA_STREAMING_CHECKS /**< Enables status checks in streaming mode (it will cause _significant_ performance penalty, max ~ 2 GB/s) */
#define IPEDMA_PAGE_SIZE 4096
#define IPEDMA_DMA_PAGES 1024 /**< number of DMA pages in the ring buffer to allocate */
#define IPEDMA_DMA_PROGRESS_THRESHOLD 1 /**< how many pages the DMA engine should fill before reporting progress */
@@ -17,7 +17,6 @@
#define IPEDMA_DESCRIPTOR_ALIGNMENT 64
-//#define IPEDMA_BUG_DMARD /**< No register read during DMA transfer */
//#define IPEDMA_BUG_LAST_READ /**< We should forbid writting the second last available DMA buffer (the last is forbidden by design) */
//#define IPEDMA_DETECT_PACKETS /**< Using empty_deceted flag */
#define IPEDMA_SUPPORT_EMPTY_DETECTED /**< Avoid waiting for data when empty_detected flag is set in hardware */
@@ -31,6 +30,7 @@
#define IPEDMA_REG_CONTROL 0x04
#define IPEDMA_REG_TLP_SIZE 0x0C
#define IPEDMA_REG_TLP_COUNT 0x10
+#define IPEDMA_REG_PCIE_GEN 0x18
#define IPEDMA_REG_PAGE_ADDR 0x50
#define IPEDMA_REG_UPDATE_ADDR 0x54
#define IPEDMA_REG_LAST_READ 0x58 /**< In streaming mode, we can use it freely to track current status */
@@ -38,6 +38,9 @@
#define IPEDMA_REG_UPDATE_THRESHOLD 0x60
#define IPEDMA_REG_STREAMING_STATUS 0x68
+#define IPEDMA_MASK_PCIE_GEN 0xF
+#define IPEDMA_MASK_STREAMING_MODE 0x10
+
#define WR(addr, value) { *(uint32_t*)(ctx->base_addr + addr) = value; }
#define RD(addr, value) { value = *(uint32_t*)(ctx->base_addr + addr); }
@@ -61,6 +64,7 @@ struct ipe_dma_s {
int reused; /**< indicates that DMA was found intialized, buffers were reused, and no additional initialization is needed */
int preserve; /**< indicates that DMA should not be stopped during clean-up */
int mode64; /**< indicates 64-bit operation mode */
+ int streaming; /**< indicates if DMA is operating in streaming or ring-buffer mode */
pcilib_kmem_handle_t *desc; /**< in-memory status descriptor written by DMA engine upon operation progess */
pcilib_kmem_handle_t *pages; /**< collection of memory-locked pages for DMA operation */
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-14 08:12:28 +0000