/alps/pcitool : contents of kmem.c at revision 117

: (revision 117)

To get this branch, use:

bzr branch
http://suren.me/webbzr/alps/pcitool

#include <stdio.h>
#include <string.h>
#include <strings.h>
#include <stdlib.h>
#include <stdint.h>
#include <stdarg.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <arpa/inet.h>
#include <errno.h>
#include <assert.h>

#include "pcilib.h"
#include "pci.h"
#include "kmem.h"
#include "error.h"

int pcilib_clean_kernel_memory(pcilib_t *ctx, pcilib_kmem_use_t use, pcilib_kmem_flags_t flags) {
    kmem_handle_t kh = {0};
    kh.use = use;
    kh.flags = flags|KMEM_FLAG_MASS;

    return ioctl(ctx->handle, PCIDRIVER_IOC_KMEM_FREE, &kh);
}


static int pcilib_free_kernel_buffer(pcilib_t *ctx, pcilib_kmem_list_t *kbuf, size_t i, pcilib_kmem_flags_t flags) {
    kmem_handle_t kh = {0};

    if (kbuf->buf.blocks[i].ua) munmap(kbuf->buf.blocks[i].ua, kbuf->buf.blocks[i].size + kbuf->buf.blocks[i].alignment_offset);
    kh.handle_id = kbuf->buf.blocks[i].handle_id;
    kh.pa = kbuf->buf.blocks[i].pa;
    kh.flags = flags;
    
    return ioctl(ctx->handle, PCIDRIVER_IOC_KMEM_FREE, &kh);
}

static void pcilib_cancel_kernel_memory(pcilib_t *ctx, pcilib_kmem_list_t *kbuf, pcilib_kmem_flags_t flags, int last_flags) {
    int ret;
    
    if (!kbuf->buf.n_blocks) return;

	// consistency error during processing of last block, special treatment could be needed
    if (last_flags) {
	pcilib_kmem_flags_t failed_flags = flags;
	
	if (last_flags&KMEM_FLAG_REUSED_PERSISTENT) flags&=~PCILIB_KMEM_FLAG_PERSISTENT;
	if (last_flags&KMEM_FLAG_REUSED_HW) flags&=~PCILIB_KMEM_FLAG_HARDWARE;
	
	if (failed_flags != flags) {
	    ret = pcilib_free_kernel_buffer(ctx, kbuf, --kbuf->buf.n_blocks, failed_flags);
	    if (ret) pcilib_error("PCIDRIVER_IOC_KMEM_FREE ioctl have failed");
	}
    }

    pcilib_free_kernel_memory(ctx, kbuf, flags);
}

pcilib_kmem_handle_t *pcilib_alloc_kernel_memory(pcilib_t *ctx, pcilib_kmem_type_t type, size_t nmemb, size_t size, size_t alignment, pcilib_kmem_use_t use, pcilib_kmem_flags_t flags) {
    int err = 0;
    const char *error = NULL;
    
    int ret;
    int i;
    void *addr;
    
    pcilib_tristate_t reused = PCILIB_TRISTATE_NO;
    int persistent = -1;
    int hardware = -1;

    kmem_handle_t kh = {0};
    
    pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)malloc(sizeof(pcilib_kmem_list_t) + nmemb * sizeof(pcilib_kmem_addr_t));
    if (!kbuf) {
	pcilib_error("Memory allocation has failed");
	return NULL;
    }
    
    memset(kbuf, 0, sizeof(pcilib_kmem_list_t) + nmemb * sizeof(pcilib_kmem_addr_t));
    

    ret = ioctl( ctx->handle, PCIDRIVER_IOC_MMAP_MODE, PCIDRIVER_MMAP_KMEM );
    if (ret) {
	pcilib_error("PCIDRIVER_IOC_MMAP_MODE ioctl have failed");
	return NULL;
    }
    
    kh.type = type;
    kh.size = size;
    kh.align = alignment;
    kh.use = use;

    if (type != PCILIB_KMEM_TYPE_PAGE) {
	kh.size += alignment;
    }
    
    for ( i = 0; i < nmemb; i++) {
	kh.item = i;
	kh.flags = flags;
	
        ret = ioctl(ctx->handle, PCIDRIVER_IOC_KMEM_ALLOC, &kh);
	if (ret) {
	    kbuf->buf.n_blocks = i;
	    error = "PCIDRIVER_IOC_KMEM_ALLOC ioctl have failed";
	    break;
	}
    
	kbuf->buf.blocks[i].handle_id = kh.handle_id;
	kbuf->buf.blocks[i].pa = kh.pa;
	kbuf->buf.blocks[i].size = kh.size;
	
	if (!i) reused = (kh.flags&KMEM_FLAG_REUSED)?PCILIB_TRISTATE_YES:PCILIB_TRISTATE_NO;

        if (kh.flags&KMEM_FLAG_REUSED) {
	    if (!i) reused = PCILIB_TRISTATE_YES;
	    else if (!reused) reused = PCILIB_TRISTATE_PARTIAL;
	
	    if (persistent) {
		if (persistent < 0) {
		    if ((flags&PCILIB_KMEM_FLAG_PERSISTENT == 0)&&(kh.flags&KMEM_FLAG_REUSED_PERSISTENT)) err = PCILIB_ERROR_INVALID_STATE;
		    else persistent = (kh.flags&KMEM_FLAG_REUSED_PERSISTENT)?1:0;
		} else if (kh.flags&KMEM_FLAG_REUSED_PERSISTENT == 0) err = PCILIB_ERROR_INVALID_STATE;
	    } else if (kh.flags&KMEM_FLAG_REUSED_PERSISTENT) err = PCILIB_ERROR_INVALID_STATE;
	    
	    if (hardware) {
		if (hardware < 0) {
		    if ((flags&PCILIB_KMEM_FLAG_HARDWARE == 0)&&(kh.flags&KMEM_FLAG_REUSED_HW)) err = PCILIB_ERROR_INVALID_STATE;
		    else hardware = (kh.flags&KMEM_FLAG_REUSED_HW)?1:0;
		} else if (kh.flags&KMEM_FLAG_REUSED_HW == 0) err = PCILIB_ERROR_INVALID_STATE;
	    } else if (kh.flags&KMEM_FLAG_REUSED_HW) err = PCILIB_ERROR_INVALID_STATE;
	    
	} else {
	    if (!i) reused = PCILIB_TRISTATE_NO;
	    else if (reused) reused = PCILIB_TRISTATE_PARTIAL;
	    
	    if ((persistent > 0)&&(flags&PCILIB_KMEM_FLAG_PERSISTENT == 0)) err = PCILIB_ERROR_INVALID_STATE;
	    if ((hardware > 0)&&(flags&PCILIB_KMEM_FLAG_HARDWARE == 0)) err = PCILIB_ERROR_INVALID_STATE;
	}
	
	if (err) {
	    kbuf->buf.n_blocks = i + 1;
	    break;
	}
    
        if ((kh.align)&&(type != PCILIB_KMEM_TYPE_PAGE)) {
	    if (kh.pa % kh.align) kbuf->buf.blocks[i].alignment_offset = kh.align - kh.pa % kh.align;
	    kbuf->buf.blocks[i].size -= kh.align;
	}
	
    	addr = mmap( 0, kbuf->buf.blocks[i].size + kbuf->buf.blocks[i].alignment_offset, PROT_WRITE | PROT_READ, MAP_SHARED, ctx->handle, 0 );
	if ((!addr)||(addr == MAP_FAILED)) {
	    kbuf->buf.n_blocks = i + 1;
	    error = "Failed to mmap allocated kernel memory";
	    break;
	}

	kbuf->buf.blocks[i].ua = addr;
//	if (use == PCILIB_KMEM_USE_DMA_PAGES) {
//	memset(addr, 10, kbuf->buf.blocks[i].size + kbuf->buf.blocks[i].alignment_offset);
//	}
	
	kbuf->buf.blocks[i].mmap_offset = kh.pa & ctx->page_mask;
    }

    if (persistent < 0) persistent = 0;
    if (hardware < 0) hardware = 0;

    if (err||error) {
	pcilib_kmem_flags_t free_flags = 0;
	
	if ((persistent <= 0)&&(flags&PCILIB_KMEM_FLAG_PERSISTENT)) {
	    free_flags |= PCILIB_KMEM_FLAG_PERSISTENT;
	}
	
	if ((hardware <= 0)&&(flags&PCILIB_KMEM_FLAG_HARDWARE)) {
	    free_flags |= PCILIB_KMEM_FLAG_HARDWARE;
	}
	
	pcilib_cancel_kernel_memory(ctx, kbuf, free_flags, err?kh.flags:0);

	if (err) error = "Reused buffers are inconsistent";
	pcilib_error(error);

	return NULL;
    }
    
    
    if (nmemb == 1) {
	memcpy(&kbuf->buf.addr, &kbuf->buf.blocks[0], sizeof(pcilib_kmem_addr_t));
    }
    
    kbuf->buf.reused = reused|(persistent?PCILIB_KMEM_REUSE_PERSISTENT:0)|(hardware?PCILIB_KMEM_REUSE_HARDWARE:0);
    kbuf->buf.n_blocks = nmemb;
    
    kbuf->prev = NULL;
    kbuf->next = ctx->kmem_list;
    if (ctx->kmem_list) ctx->kmem_list->prev = kbuf;
    ctx->kmem_list = kbuf;

    return (pcilib_kmem_handle_t*)kbuf;
}

void pcilib_free_kernel_memory(pcilib_t *ctx, pcilib_kmem_handle_t *k, pcilib_kmem_flags_t flags) {
    int ret, err = 0; 
    int i;
    kmem_handle_t kh = {0};
    pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)k;

	// if linked in to the list
    if (kbuf->next) kbuf->next->prev = kbuf->prev;
    if (kbuf->prev) kbuf->prev->next = kbuf->next;
    else if (ctx->kmem_list == kbuf) ctx->kmem_list = kbuf->next;

    for (i = 0; i < kbuf->buf.n_blocks; i++) {
        ret = pcilib_free_kernel_buffer(ctx, kbuf, i, flags);
    	if ((ret)&&(!err)) err = ret;
    }
    
    free(kbuf);
    
    if (err) {
	pcilib_error("PCIDRIVER_IOC_KMEM_FREE ioctl have failed");
    }
}

/*
int pcilib_sync_kernel_memory(pcilib_t *ctx, pcilib_kmem_handle_t *k, pcilib_kmem_sync_direction_t dir) {
    int i;
    int ret;
    kmem_sync_t ks;
    pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)k;
    
    ks.dir = dir;

    for (i = 0; i < kbuf->buf.n_blocks; i++) {
        ks.handle.handle_id = kbuf->buf.blocks[i].handle_id;
	ks.handle.pa = kbuf->buf.blocks[i].pa;
	ret = ioctl(ctx->handle, PCIDRIVER_IOC_KMEM_SYNC, &ks);
	if (ret) {
	    pcilib_error("PCIDRIVER_IOC_KMEM_SYNC ioctl have failed");
	    return PCILIB_ERROR_FAILED;
	}
    }
    
    return 0;    
}

*/

int pcilib_kmem_sync_block(pcilib_t *ctx, pcilib_kmem_handle_t *k, pcilib_kmem_sync_direction_t dir, size_t block) {
    int ret;
    kmem_sync_t ks;
    pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)k;

    ks.dir = dir;
    ks.handle.handle_id = kbuf->buf.blocks[block].handle_id;
    ks.handle.pa = kbuf->buf.blocks[block].pa;
    ret = ioctl(ctx->handle, PCIDRIVER_IOC_KMEM_SYNC, &ks);
    if (ret) {
	pcilib_error("PCIDRIVER_IOC_KMEM_SYNC ioctl have failed");
	return PCILIB_ERROR_FAILED;
    }
    
    return 0;
}

void *pcilib_kmem_get_ua(pcilib_t *ctx, pcilib_kmem_handle_t *k) {
    pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)k;
    return kbuf->buf.addr.ua + kbuf->buf.addr.alignment_offset + kbuf->buf.addr.mmap_offset;
}

uintptr_t pcilib_kmem_get_pa(pcilib_t *ctx, pcilib_kmem_handle_t *k) {
    pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)k;
    return kbuf->buf.addr.pa + kbuf->buf.addr.alignment_offset;
}

void *pcilib_kmem_get_block_ua(pcilib_t *ctx, pcilib_kmem_handle_t *k, size_t block) {
    pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)k;
    return kbuf->buf.blocks[block].ua + kbuf->buf.blocks[block].alignment_offset + kbuf->buf.blocks[block].mmap_offset;
}

uintptr_t pcilib_kmem_get_block_pa(pcilib_t *ctx, pcilib_kmem_handle_t *k, size_t block) {
    pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)k;
    return kbuf->buf.blocks[block].pa + kbuf->buf.blocks[block].alignment_offset;
}

size_t pcilib_kmem_get_block_size(pcilib_t *ctx, pcilib_kmem_handle_t *k, size_t block) {
    pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)k;
    return kbuf->buf.blocks[block].size;
}

pcilib_kmem_reuse_state_t  pcilib_kmem_is_reused(pcilib_t *ctx, pcilib_kmem_handle_t *k) {
    pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)k;
    return kbuf->buf.reused;
}

45 by root North West Logick DMA implementation	1	#include <stdio.h>
	2	#include <string.h>
	3	#include <strings.h>
	4	#include <stdlib.h>
	5	#include <stdint.h>
	6	#include <stdarg.h>
	7	#include <fcntl.h>
	8	#include <unistd.h>
	9	#include <sys/ioctl.h>
	10	#include <sys/mman.h>
	11	#include <arpa/inet.h>
	12	#include <errno.h>
	13	#include <assert.h>
	14
	15	#include "pcilib.h"
	16	#include "pci.h"
	17	#include "kmem.h"
	18	#include "error.h"
	19
81 by Suren A. Chilingaryan Support forceful clean-up of kernel memory	20	int pcilib_clean_kernel_memory(pcilib_t *ctx, pcilib_kmem_use_t use, pcilib_kmem_flags_t flags) {
	21	kmem_handle_t kh = {0};
	22	kh.use = use;
	23	kh.flags = flags\|KMEM_FLAG_MASS;
	24
	25	return ioctl(ctx->handle, PCIDRIVER_IOC_KMEM_FREE, &kh);
	26	}
	27
	28
74 by Suren A. Chilingaryan Implement DMA access synchronization for NWL implementation	29	static int pcilib_free_kernel_buffer(pcilib_t ctx, pcilib_kmem_list_t kbuf, size_t i, pcilib_kmem_flags_t flags) {
	30	kmem_handle_t kh = {0};
	31
	32	if (kbuf->buf.blocks[i].ua) munmap(kbuf->buf.blocks[i].ua, kbuf->buf.blocks[i].size + kbuf->buf.blocks[i].alignment_offset);
	33	kh.handle_id = kbuf->buf.blocks[i].handle_id;
	34	kh.pa = kbuf->buf.blocks[i].pa;
	35	kh.flags = flags;
75 by Suren A. Chilingaryan Few fixes	36
74 by Suren A. Chilingaryan Implement DMA access synchronization for NWL implementation	37	return ioctl(ctx->handle, PCIDRIVER_IOC_KMEM_FREE, &kh);
	38	}
	39
	40	static void pcilib_cancel_kernel_memory(pcilib_t ctx, pcilib_kmem_list_t kbuf, pcilib_kmem_flags_t flags, int last_flags) {
	41	int ret;
	42
	43	if (!kbuf->buf.n_blocks) return;
	44
	45	// consistency error during processing of last block, special treatment could be needed
	46	if (last_flags) {
	47	pcilib_kmem_flags_t failed_flags = flags;
	48
	49	if (last_flags&KMEM_FLAG_REUSED_PERSISTENT) flags&=~PCILIB_KMEM_FLAG_PERSISTENT;
	50	if (last_flags&KMEM_FLAG_REUSED_HW) flags&=~PCILIB_KMEM_FLAG_HARDWARE;
	51
	52	if (failed_flags != flags) {
	53	ret = pcilib_free_kernel_buffer(ctx, kbuf, --kbuf->buf.n_blocks, failed_flags);
	54	if (ret) pcilib_error("PCIDRIVER_IOC_KMEM_FREE ioctl have failed");
	55	}
	56	}
	57
	58	pcilib_free_kernel_memory(ctx, kbuf, flags);
	59	}
	60
45 by root North West Logick DMA implementation	61	pcilib_kmem_handle_t pcilib_alloc_kernel_memory(pcilib_t ctx, pcilib_kmem_type_t type, size_t nmemb, size_t size, size_t alignment, pcilib_kmem_use_t use, pcilib_kmem_flags_t flags) {
73 by Suren A. Chilingaryan Implement DMA access synchronization in the driver	62	int err = 0;
	63	const char *error = NULL;
	64
45 by root North West Logick DMA implementation	65	int ret;
	66	int i;
	67	void *addr;
71 by Suren A. Chilingaryan First iteration of work to preserve DMA state between executions	68
73 by Suren A. Chilingaryan Implement DMA access synchronization in the driver	69	pcilib_tristate_t reused = PCILIB_TRISTATE_NO;
	70	int persistent = -1;
	71	int hardware = -1;
45 by root North West Logick DMA implementation	72
	73	kmem_handle_t kh = {0};
	74
	75	pcilib_kmem_list_t kbuf = (pcilib_kmem_list_t)malloc(sizeof(pcilib_kmem_list_t) + nmemb * sizeof(pcilib_kmem_addr_t));
	76	if (!kbuf) {
	77	pcilib_error("Memory allocation has failed");
	78	return NULL;
	79	}
	80
	81	memset(kbuf, 0, sizeof(pcilib_kmem_list_t) + nmemb * sizeof(pcilib_kmem_addr_t));
	82
	83
	84	ret = ioctl( ctx->handle, PCIDRIVER_IOC_MMAP_MODE, PCIDRIVER_MMAP_KMEM );
	85	if (ret) {
	86	pcilib_error("PCIDRIVER_IOC_MMAP_MODE ioctl have failed");
	87	return NULL;
	88	}
	89
	90	kh.type = type;
	91	kh.size = size;
	92	kh.align = alignment;
	93	kh.use = use;
	94
52 by Suren A. Chilingaryan Support alignments in kmem allocation	95	if (type != PCILIB_KMEM_TYPE_PAGE) {
	96	kh.size += alignment;
	97	}
	98
45 by root North West Logick DMA implementation	99	for ( i = 0; i < nmemb; i++) {
71 by Suren A. Chilingaryan First iteration of work to preserve DMA state between executions	100	kh.item = i;
73 by Suren A. Chilingaryan Implement DMA access synchronization in the driver	101	kh.flags = flags;
	102
45 by root North West Logick DMA implementation	103	ret = ioctl(ctx->handle, PCIDRIVER_IOC_KMEM_ALLOC, &kh);
	104	if (ret) {
	105	kbuf->buf.n_blocks = i;
73 by Suren A. Chilingaryan Implement DMA access synchronization in the driver	106	error = "PCIDRIVER_IOC_KMEM_ALLOC ioctl have failed";
	107	break;
45 by root North West Logick DMA implementation	108	}
	109
	110	kbuf->buf.blocks[i].handle_id = kh.handle_id;
	111	kbuf->buf.blocks[i].pa = kh.pa;
	112	kbuf->buf.blocks[i].size = kh.size;
73 by Suren A. Chilingaryan Implement DMA access synchronization in the driver	113
	114	if (!i) reused = (kh.flags&KMEM_FLAG_REUSED)?PCILIB_TRISTATE_YES:PCILIB_TRISTATE_NO;
	115
	116	if (kh.flags&KMEM_FLAG_REUSED) {
	117	if (!i) reused = PCILIB_TRISTATE_YES;
	118	else if (!reused) reused = PCILIB_TRISTATE_PARTIAL;
	119
	120	if (persistent) {
74 by Suren A. Chilingaryan Implement DMA access synchronization for NWL implementation	121	if (persistent < 0) {
	122	if ((flags&PCILIB_KMEM_FLAG_PERSISTENT == 0)&&(kh.flags&KMEM_FLAG_REUSED_PERSISTENT)) err = PCILIB_ERROR_INVALID_STATE;
	123	else persistent = (kh.flags&KMEM_FLAG_REUSED_PERSISTENT)?1:0;
	124	} else if (kh.flags&KMEM_FLAG_REUSED_PERSISTENT == 0) err = PCILIB_ERROR_INVALID_STATE;
73 by Suren A. Chilingaryan Implement DMA access synchronization in the driver	125	} else if (kh.flags&KMEM_FLAG_REUSED_PERSISTENT) err = PCILIB_ERROR_INVALID_STATE;
	126
	127	if (hardware) {
74 by Suren A. Chilingaryan Implement DMA access synchronization for NWL implementation	128	if (hardware < 0) {
	129	if ((flags&PCILIB_KMEM_FLAG_HARDWARE == 0)&&(kh.flags&KMEM_FLAG_REUSED_HW)) err = PCILIB_ERROR_INVALID_STATE;
	130	else hardware = (kh.flags&KMEM_FLAG_REUSED_HW)?1:0;
	131	} else if (kh.flags&KMEM_FLAG_REUSED_HW == 0) err = PCILIB_ERROR_INVALID_STATE;
73 by Suren A. Chilingaryan Implement DMA access synchronization in the driver	132	} else if (kh.flags&KMEM_FLAG_REUSED_HW) err = PCILIB_ERROR_INVALID_STATE;
	133
	134	} else {
	135	if (!i) reused = PCILIB_TRISTATE_NO;
	136	else if (reused) reused = PCILIB_TRISTATE_PARTIAL;
74 by Suren A. Chilingaryan Implement DMA access synchronization for NWL implementation	137
	138	if ((persistent > 0)&&(flags&PCILIB_KMEM_FLAG_PERSISTENT == 0)) err = PCILIB_ERROR_INVALID_STATE;
	139	if ((hardware > 0)&&(flags&PCILIB_KMEM_FLAG_HARDWARE == 0)) err = PCILIB_ERROR_INVALID_STATE;
71 by Suren A. Chilingaryan First iteration of work to preserve DMA state between executions	140	}
73 by Suren A. Chilingaryan Implement DMA access synchronization in the driver	141
74 by Suren A. Chilingaryan Implement DMA access synchronization for NWL implementation	142	if (err) {
	143	kbuf->buf.n_blocks = i + 1;
	144	break;
	145	}
52 by Suren A. Chilingaryan Support alignments in kmem allocation	146
100 by root Support exporting data from kernel buffers	147	if ((kh.align)&&(type != PCILIB_KMEM_TYPE_PAGE)) {
	148	if (kh.pa % kh.align) kbuf->buf.blocks[i].alignment_offset = kh.align - kh.pa % kh.align;
	149	kbuf->buf.blocks[i].size -= kh.align;
52 by Suren A. Chilingaryan Support alignments in kmem allocation	150	}
45 by root North West Logick DMA implementation	151
76 by Suren A. Chilingaryan Handle correctly reference counting in the driver	152	addr = mmap( 0, kbuf->buf.blocks[i].size + kbuf->buf.blocks[i].alignment_offset, PROT_WRITE \| PROT_READ, MAP_SHARED, ctx->handle, 0 );
45 by root North West Logick DMA implementation	153	if ((!addr)\|\|(addr == MAP_FAILED)) {
45 by root North West Logick DMA implementation	154	kbuf->buf.n_blocks = i + 1;
73 by Suren A. Chilingaryan Implement DMA access synchronization in the driver	155	error = "Failed to mmap allocated kernel memory";
	156	break;
45 by root North West Logick DMA implementation	157	}
	158
	159	kbuf->buf.blocks[i].ua = addr;
100 by root Support exporting data from kernel buffers	160	// if (use == PCILIB_KMEM_USE_DMA_PAGES) {
	161	// memset(addr, 10, kbuf->buf.blocks[i].size + kbuf->buf.blocks[i].alignment_offset);
	162	// }
52 by Suren A. Chilingaryan Support alignments in kmem allocation	163
	164	kbuf->buf.blocks[i].mmap_offset = kh.pa & ctx->page_mask;
45 by root North West Logick DMA implementation	165	}
73 by Suren A. Chilingaryan Implement DMA access synchronization in the driver	166
74 by Suren A. Chilingaryan Implement DMA access synchronization for NWL implementation	167	if (persistent < 0) persistent = 0;
	168	if (hardware < 0) hardware = 0;
73 by Suren A. Chilingaryan Implement DMA access synchronization in the driver	169
	170	if (err\|\|error) {
	171	pcilib_kmem_flags_t free_flags = 0;
	172
74 by Suren A. Chilingaryan Implement DMA access synchronization for NWL implementation	173	if ((persistent <= 0)&&(flags&PCILIB_KMEM_FLAG_PERSISTENT)) {
73 by Suren A. Chilingaryan Implement DMA access synchronization in the driver	174	free_flags \|= PCILIB_KMEM_FLAG_PERSISTENT;
	175	}
74 by Suren A. Chilingaryan Implement DMA access synchronization for NWL implementation	176
	177	if ((hardware <= 0)&&(flags&PCILIB_KMEM_FLAG_HARDWARE)) {
73 by Suren A. Chilingaryan Implement DMA access synchronization in the driver	178	free_flags \|= PCILIB_KMEM_FLAG_HARDWARE;
	179	}
	180
74 by Suren A. Chilingaryan Implement DMA access synchronization for NWL implementation	181	pcilib_cancel_kernel_memory(ctx, kbuf, free_flags, err?kh.flags:0);
73 by Suren A. Chilingaryan Implement DMA access synchronization in the driver	182
	183	if (err) error = "Reused buffers are inconsistent";
	184	pcilib_error(error);
	185
	186	return NULL;
	187	}
	188
45 by root North West Logick DMA implementation	189
	190	if (nmemb == 1) {
	191	memcpy(&kbuf->buf.addr, &kbuf->buf.blocks[0], sizeof(pcilib_kmem_addr_t));
	192	}
	193
73 by Suren A. Chilingaryan Implement DMA access synchronization in the driver	194	kbuf->buf.reused = reused\|(persistent?PCILIB_KMEM_REUSE_PERSISTENT:0)\|(hardware?PCILIB_KMEM_REUSE_HARDWARE:0);
45 by root North West Logick DMA implementation	195	kbuf->buf.n_blocks = nmemb;
	196
	197	kbuf->prev = NULL;
	198	kbuf->next = ctx->kmem_list;
	199	if (ctx->kmem_list) ctx->kmem_list->prev = kbuf;
	200	ctx->kmem_list = kbuf;
	201
	202	return (pcilib_kmem_handle_t*)kbuf;
	203	}
	204
71 by Suren A. Chilingaryan First iteration of work to preserve DMA state between executions	205	void pcilib_free_kernel_memory(pcilib_t ctx, pcilib_kmem_handle_t k, pcilib_kmem_flags_t flags) {
45 by root North West Logick DMA implementation	206	int ret, err = 0;
	207	int i;
	208	kmem_handle_t kh = {0};
	209	pcilib_kmem_list_t kbuf = (pcilib_kmem_list_t)k;
	210
	211	// if linked in to the list
	212	if (kbuf->next) kbuf->next->prev = kbuf->prev;
	213	if (kbuf->prev) kbuf->prev->next = kbuf->next;
	214	else if (ctx->kmem_list == kbuf) ctx->kmem_list = kbuf->next;
	215
	216	for (i = 0; i < kbuf->buf.n_blocks; i++) {
75 by Suren A. Chilingaryan Few fixes	217	ret = pcilib_free_kernel_buffer(ctx, kbuf, i, flags);
74 by Suren A. Chilingaryan Implement DMA access synchronization for NWL implementation	218	if ((ret)&&(!err)) err = ret;
45 by root North West Logick DMA implementation	219	}
	220
	221	free(kbuf);
	222
	223	if (err) {
	224	pcilib_error("PCIDRIVER_IOC_KMEM_FREE ioctl have failed");
	225	}
	226	}
	227
106 by Suren A. Chilingaryan Sync only required buffers	228	/*
45 by root North West Logick DMA implementation	229	int pcilib_sync_kernel_memory(pcilib_t ctx, pcilib_kmem_handle_t k, pcilib_kmem_sync_direction_t dir) {
	230	int i;
	231	int ret;
	232	kmem_sync_t ks;
	233	pcilib_kmem_list_t kbuf = (pcilib_kmem_list_t)k;
	234
	235	ks.dir = dir;
106 by Suren A. Chilingaryan Sync only required buffers	236
45 by root North West Logick DMA implementation	237	for (i = 0; i < kbuf->buf.n_blocks; i++) {
	238	ks.handle.handle_id = kbuf->buf.blocks[i].handle_id;
	239	ks.handle.pa = kbuf->buf.blocks[i].pa;
	240	ret = ioctl(ctx->handle, PCIDRIVER_IOC_KMEM_SYNC, &ks);
	241	if (ret) {
	242	pcilib_error("PCIDRIVER_IOC_KMEM_SYNC ioctl have failed");
	243	return PCILIB_ERROR_FAILED;
	244	}
	245	}
	246
	247	return 0;
	248	}
	249
106 by Suren A. Chilingaryan Sync only required buffers	250	*/
	251
	252	int pcilib_kmem_sync_block(pcilib_t ctx, pcilib_kmem_handle_t k, pcilib_kmem_sync_direction_t dir, size_t block) {
	253	int ret;
	254	kmem_sync_t ks;
	255	pcilib_kmem_list_t kbuf = (pcilib_kmem_list_t)k;
	256
	257	ks.dir = dir;
	258	ks.handle.handle_id = kbuf->buf.blocks[block].handle_id;
	259	ks.handle.pa = kbuf->buf.blocks[block].pa;
	260	ret = ioctl(ctx->handle, PCIDRIVER_IOC_KMEM_SYNC, &ks);
	261	if (ret) {
	262	pcilib_error("PCIDRIVER_IOC_KMEM_SYNC ioctl have failed");
	263	return PCILIB_ERROR_FAILED;
	264	}
	265
	266	return 0;
	267	}
45 by root North West Logick DMA implementation	268
	269	void pcilib_kmem_get_ua(pcilib_t ctx, pcilib_kmem_handle_t *k) {
	270	pcilib_kmem_list_t kbuf = (pcilib_kmem_list_t)k;
52 by Suren A. Chilingaryan Support alignments in kmem allocation	271	return kbuf->buf.addr.ua + kbuf->buf.addr.alignment_offset + kbuf->buf.addr.mmap_offset;
45 by root North West Logick DMA implementation	272	}
	273
	274	uintptr_t pcilib_kmem_get_pa(pcilib_t ctx, pcilib_kmem_handle_t k) {
	275	pcilib_kmem_list_t kbuf = (pcilib_kmem_list_t)k;
52 by Suren A. Chilingaryan Support alignments in kmem allocation	276	return kbuf->buf.addr.pa + kbuf->buf.addr.alignment_offset;
45 by root North West Logick DMA implementation	277	}
	278
	279	void pcilib_kmem_get_block_ua(pcilib_t ctx, pcilib_kmem_handle_t *k, size_t block) {
	280	pcilib_kmem_list_t kbuf = (pcilib_kmem_list_t)k;
52 by Suren A. Chilingaryan Support alignments in kmem allocation	281	return kbuf->buf.blocks[block].ua + kbuf->buf.blocks[block].alignment_offset + kbuf->buf.blocks[block].mmap_offset;
45 by root North West Logick DMA implementation	282	}
	283
	284	uintptr_t pcilib_kmem_get_block_pa(pcilib_t ctx, pcilib_kmem_handle_t k, size_t block) {
	285	pcilib_kmem_list_t kbuf = (pcilib_kmem_list_t)k;
52 by Suren A. Chilingaryan Support alignments in kmem allocation	286	return kbuf->buf.blocks[block].pa + kbuf->buf.blocks[block].alignment_offset;
45 by root North West Logick DMA implementation	287	}
	288
	289	size_t pcilib_kmem_get_block_size(pcilib_t ctx, pcilib_kmem_handle_t k, size_t block) {
	290	pcilib_kmem_list_t kbuf = (pcilib_kmem_list_t)k;
	291	return kbuf->buf.blocks[block].size;
	292	}
71 by Suren A. Chilingaryan First iteration of work to preserve DMA state between executions	293
73 by Suren A. Chilingaryan Implement DMA access synchronization in the driver	294	pcilib_kmem_reuse_state_t pcilib_kmem_is_reused(pcilib_t ctx, pcilib_kmem_handle_t k) {
71 by Suren A. Chilingaryan First iteration of work to preserve DMA state between executions	295	pcilib_kmem_list_t kbuf = (pcilib_kmem_list_t)k;
73 by Suren A. Chilingaryan Implement DMA access synchronization in the driver	296	return kbuf->buf.reused;
71 by Suren A. Chilingaryan First iteration of work to preserve DMA state between executions	297	}