T4240 fsl-usdpaa memory,write file by O_DIRECT failed

[silecy2016]

I use Device Drivers ->Userspace I/O drivers ->Freescale Serial RapidIO support, then in linux, I get /dev/fsl-usdpaa. I use "dma_mem_create" to get continuous memory, and use this memory to write file by O_DIRECT. But it return -14, bad address. If I use "aligned_alloc" to get memory, it is OK to write file by O_DIRECT. How can I get continuous memory(more than 2GB) and write file by O_DIRECT.

following is result:

events -14 0
k = 2 ix = 2

$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$
#include <stdlib.h>
#include <stdio.h>
#define __USE_LARGEFILE64
#define _FILE_OFFSET_BITS 64
#define __USE_GNU
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <time.h>
#include <libaio.h>

#define MB (0x100000ULL)
#define GB (1024*MB)
#define FILE_SIZE (2*MB)
#define NUM_IOCB (FILE_SIZE/MB)

io_context_t ioctx;
unsigned char DATA[MB];
struct iocb iocbs[NUM_IOCB];
struct io_event events[32];

int main(int argc, const char *argv)
{
struct iocb *iocb_array, *piocb[32];
int fd, err, ret;
long long off=0;
int n=0,i=0,x=0,ix=0, k=0;
struct timespec start,end;
unsigned long long start_ns, end_ns;
unsigned char *buff;

ddrMap(0, 0);

fd = open("/mnt/a.bin", O_RDWR|O_CREAT|O_DIRECT|O_LARGEFILE, 0666);
if (fd < 0) {
printf("open() failed\n");
return -1;
}
#if 0
err = fallocate64(fd, 0, 0, FILE_SIZE);
if (err != 0) {
printf("fallocate64() failed\n");
return -1;
}
#endif
err = io_setup(1024, &ioctx);
if (err != 0) {
perror("io_setup");
printf("io_setup() failed\n");
return -1;
}

iocb_array = malloc(NUM_IOCB*sizeof(struct iocb));
if (iocb_array == NULL) {
printf("malloc() failed\n");
return -1;
}

buff = (void*)aligned_alloc(MB, MB);
if (buff == NULL) {
printf("aligned_alloc() failed\n");
return -1;
}
buff = getDataBaseAddr();

printf("buff = %lx\n", (unsigned long int)buff);

memset(iocb_array, 0, sizeof(NUM_IOCB*sizeof(struct iocb)));
for (i = 0; i < NUM_IOCB; i++) {
io_prep_pwrite(&iocb_array[i], fd, buff, MB, off);
off+=MB;
}

memset(buff, 0x12, 0x100000);
clock_gettime(CLOCK_MONOTONIC, &start);
start_ns = start.tv_sec*1000000000llu+start.tv_nsec;

x = ix = k = 0;

while (ix < NUM_IOCB) {
/* x：当前已经转发的数量, ix数组下标, i待转发的数量 */
for (i = 0; x < 2; x++, i++, ix++)
piocb[i] = &iocb_array[ix];

if (i > 0) {
ret = io_submit(ioctx, i, piocb);
if (ret != i) {

printf("io_submit() failed i = %d errno = %d\n", i, ret);
return -1;
}
}

ret = io_getevents(ioctx, 1, 32, events, NULL);
if (ret > 0) {
printf("events %d %d\n", events[0].res, events[0].res2);
k += ret;
x-=ret;
if (k%1024 == 0) {
clock_gettime(CLOCK_MONOTONIC, &end);
end_ns = end.tv_sec*1000000000llu+end.tv_nsec;
printf("time: %lld\n", (end_ns-start_ns)/(1000*1000));
}
}
}

printf("k = %d ix = %d\n", k, ix);
}

$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$

#include <stdlib.h>
#include <stdio.h>
#include <stdint.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <dma_mem.h>

#define MEM_LEN 0x1000000ULL /* 1GB 字节 */

static unsigned long int resAddr = 0;
static unsigned long int resPhyAddr = 0;
static unsigned long int dataAddr = 0;
static unsigned long int dataPhyAddr = 0;
static unsigned long int dataSize = 0;

unsigned long int getResBaseAddr(void)
{
return resAddr;
}

unsigned long int getResBasePhyAddr(void)
{
return resPhyAddr;
}

unsigned long int getDataBaseAddr(void)
{
return dataAddr;
}

unsigned long int getDataBasePhyAddr(void)
{
return dataPhyAddr;
}

unsigned long int getDataSize(void)
{
return dataSize;
}

void *checkMemory(void *arg)
{
unsigned long int *pt = (unsigned long int*)(arg);
unsigned long int *pt2 = (unsigned long int*)(arg);

printf("0x%lx : 0x%lx 0x%lx 0x%lx 0x%lx", (unsigned long int)arg,
*pt, *(pt+1), *(pt+2), *(pt+3));
return NULL;
}

void *memoryC(void *arg)
{
int i = 0;
while(1)
{
sleep(1);
checkMemory(0x20000000 + i*0x100000);
printf("\t");
checkMemory(0x30000000 + i*0x100000);
printf("\n");
i++;
if (i >= 256)
{
i = 0;
}
}
}

int startFra(void)
{
system("/home/root/fra -r -f /home/root/db.xml");
}

int ddrMap(int fill, int fra)
{
#if 0

int flags = 0;
unsigned long int len;
char str[32];
int ret;
struct dma_mem *mydma[16];
unsigned long int *tmp;
unsigned char *pdata;
int i;
pthread_t fraHandle;
flags |= DMA_MAP_FLAG_SHARED;
flags |= DMA_MAP_FLAG_NEW;

flags = 0x10;

printf("配置内存：%sMB \n", "256");
len = 1;
for (i=0; i<len; i++)
{
sprintf(str, "ssd%d", i);
mydma[i] = dma_mem_create(flags, NULL, MEM_LEN);
if (mydma[i] == NULL)
{
perror("dma_mem_create");
printf("申请第%d个内存块失败!\n", i);
exit(0);
}
}

for (i=0; i<len; i++)
{
tmp = (unsigned long int*)mydma[i];
dataAddr = *(tmp+1);
dataPhyAddr = *(tmp);
printf("内存映射：物理地址 0x%lx --> 逻辑地址 0x%lx 大小 0x%x Bytes\n", *(tmp), *(tmp+1), MEM_LEN);
}

/* 保留区域的地址 */
tmp = (unsigned long int*)mydma[0];
resAddr = *(tmp+1);
resPhyAddr = *(tmp);

/* 数据区域的地址和大小 */
dataSize = MEM_LEN;
tmp = (unsigned long int*)mydma[0];
dataAddr = *(tmp+1);
tmp = (unsigned long int*)mydma[0];
dataPhyAddr = *(tmp);

/* 启动FRA程序 */
// if (fra == 1)
// {
// pthread_create(&fraHandle, NULL, startFra, NULL);
// }

return 0;

#else

int i;
int fd_mem;
pthread_t fraHandle;
unsigned long int ddrStart = 0;
unsigned long int ddrPhyAddress = 0x300000000ULL;
unsigned long int ddrSize = 0x40000000ULL;
unsigned int *tmp;
fd_mem = open("/dev/mem", O_RDWR|O_SYNC);
if (fd_mem < 0)
{
printf("open /dev/mem faild\n");
return -1;
}

ddrStart = (long unsigned int)mmap64((void*)ddrPhyAddress, ddrSize,
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_LOCKED,
fd_mem, ddrPhyAddress);
if ((void *)ddrStart == MAP_FAILED)
{
close(fd_mem);
printf("mmap rio regs failed\n");
return -1;
}

printf("DDR MAP : 0x%lx --> 0x%lx\n",
(long unsigned int)ddrPhyAddress, (long unsigned int)ddrStart);
close(fd_mem);

// tmp = (unsigned int *)ddrStart;
// for (i=0; i<0x4000000; i++)
// {
// *(tmp+i) = i+1;
// }

dataPhyAddr = 0xc0000000ULL;
dataAddr = ddrStart;
// /* 启动FRA程序 */
// if (fra == 1)
// {
// pthread_create(&fraHandle, NULL, startFra, NULL);
// }

return 0;
#endif
}