/* Guochun Shi <gshi@ncsa.uiuc.edu>
 */

#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <libspe.h>
#include "spu/speinfo.h"
#include <unistd.h>
#include <ctype.h>
#include <sys/time.h>
#include <string.h>

#include <stdlib.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/ipc.h>
#include <sys/shm.h>
#include <sys/mman.h>
#include <fcntl.h>

extern spe_program_handle_t simple_spu;

#define SPU_THREADS 	8

#define COUNT  (1024*1024*16*2)

#ifndef __HUGETLBFS__
char bufa[COUNT] __attribute__ ((aligned (128)));
char bufc[COUNT] __attribute__ ((aligned (128)));

int alloc_mem()
{
    return 0;
}

void dealloc_mem()
{
    return;
}

#else

char* bufa;
char* bufc;
int fd;

#define FILE_NAME "/mnt/huge/hugefile"
int alloc_mem(){
    char* addr;
    
    printf("Using hugetlbfs \n");
    fd = open(FILE_NAME, O_CREAT|O_RDWR, 0755);
    if(fd < 0){
	perror("Open failed\n");
	exit(1);
    }
    addr  = mmap(0, COUNT*2, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
    if (addr ==MAP_FAILED){
	perror("mmap");
	unlink(FILE_NAME);
	exit(1);
    }
    

    bufa = addr;
    bufc =  addr + COUNT;
    printf("addr =%p, bufc=%p\n", addr, bufc);    
    return 0;
}

void
dealloc_mem()
{
    munmap(bufa, COUNT*2);
    close(fd);
    unlink(FILE_NAME);
}

#endif

float  gettime_print(int num)
{
    static struct timeval oldt={0,0};
    struct timeval t;
    double result;

    if (gettimeofday(&t, NULL) < 0){
        fprintf(stderr, "ERROR: gettimeofday() failed\n");
        return -1;
    }
    if (oldt.tv_sec == 0 && oldt.tv_usec ==0){
        result = 0;
    }else {
        result = t.tv_sec - oldt.tv_sec + (t.tv_usec-oldt.tv_usec)*0.000001;
    }

    oldt = t;
    if (num !=0){
	printf("#%d time: %f seconds\n",num, result);
    }
    return result;


}

void
array_compare(char* m1, char* m2)
{
    int errcount =0;
    int i;
    
    for (i = 0; i< COUNT ; i++){
	if (m1[i] != m2[i]){
	    printf("i=%d, m1=%d, m2=%d\n",  i, m1[i], m2[i]);
	    errcount ++;
	    //exit(1);
	}
	
    }
    if (errcount == 0){
	printf("passed\n");
    }else{
	printf("failed(%d)\n", errcount);
	exit(1);
    }
    return;

}


int init_value(void)
{
    int i;
	
    printf("bufa =%p, bufc=%p\n", bufa, bufc);

    memset(bufc, 0, COUNT*sizeof(char));    
    

    for (i = 0; i< COUNT;i++){
       bufa[i] = rand() %  256;
    }
    
    return 0;
        
}



void
usage(char* prog){
    printf("%s: -n <num> -v \n", prog);
    return;
}
int need_verify = 0;
int g_spe_threads ;
int repeat_num=2;
int
processopt(int argc, char **argv)
{
    char *cvalue = NULL;
    int n;
    signed char c;
    opterr = 0;

    n = 1;/*default number of spus*/
    
    while ((c = getopt (argc, argv, "n:vr:")) != -1){
        switch (c)
            {
            case 'n':
                cvalue = optarg;
                if (sscanf(cvalue, "%d", &n) ==0 || n < 0){
                    fprintf(stderr, "Invalid spu number(%d)\n", n);
                    return -1;
                }
                break;
	    case 'r':
		cvalue=optarg;
                if (sscanf(cvalue, "%d", &repeat_num) ==0 || repeat_num < 0){
                    fprintf(stderr, "Invalid repeat num(%d)\n", repeat_num);
                    return -1;
                }
                break;
		
            case 'v':
		need_verify = 1;
                break;
            case '?':
                if (isprint (optopt))
                    fprintf (stderr, "Unknown option `-%c'.\n", optopt);
                else
                    fprintf (stderr,
                             "Unknown option character `\\x%x'.\n",
                             optopt);
                usage(argv[0]);
                return - 1;
            default:
                usage(argv[0]);
                return -1;
            }
    }
     
    g_spe_threads = n;
    printf("Program running on %d SPUs\n", n);
    return 0;
}



int main(int argc, char **argv)
{
    speid_t spe_ids[SPU_THREADS];
    int i;
    struct spe_info si[SPU_THREADS] __attribute__ ((aligned (128)));
    
    int totalcount =  COUNT; /* number of matrces*/
    int num_runs = 0 ;
    float totaltime = 0;
    float bandwidth ;
    float ave_time;

    (void)argc;
    (void)argv;
    
   

    
    processopt(argc,argv);
    
    
    alloc_mem();

    for(i=0; i<g_spe_threads; i++){
	spe_ids[i] = spe_create_thread(0, &simple_spu, &si[0]  , NULL, -1, SPE_MAP_PS);
	if (spe_ids[i] == 0) {
	    fprintf(stderr, "Failed spu_create_thread(rc=%p, errno=%d)\n", 
                spe_ids[i], errno);
	exit(1);
	}
      
    }

    /* fill the spe_info struct*/
    for (i = 0; i < g_spe_threads; i++){

	si[i].rank = i;
	si[i].speid = (unsigned long)spe_ids[i];
	si[i].count = totalcount/g_spe_threads;
	si[i].bufa = (unsigned long)bufa;
	si[i].bufc = (unsigned long)bufc;
	
    }
    
    
    for (i = 0; i < g_spe_threads; i++){
	int phys_id ;
	if (spe_write_in_mbox(spe_ids[i], g_spe_threads) < 0
	    || spe_write_in_mbox(spe_ids[i], si[i].rank) < 0){
	    fprintf(stderr, "ERROR: writing messages to spe failed\n");
	    exit(1);
	}
	//phys_id = spe_get_phys_id(spe_ids[i]);
	//printf("phys id =0x%x\n", phys_id);
    }


    init_value();
    
 repeat:

    gettime_print(0);
    
    if (g_spe_threads == 0){
	
    }else{
	for (i = 0; i < g_spe_threads; i++){
	    spe_write_in_mbox(spe_ids[i], 1);
	}
	
	for (i =0; i < g_spe_threads; i++){
	    while(spe_read_out_mbox(spe_ids[i])!= 1)
		{;}
	    
	}


    }

    totaltime += gettime_print(1);	    
    
    if (need_verify){
	printf("verifying data ...");
	array_compare(bufa, bufc);
    }
    if (++num_runs < repeat_num){
	goto repeat;
    }
    
    dealloc_mem();
    ave_time = totaltime /num_runs;
    printf("average time is =%f\n", ave_time);

    bandwidth = (COUNT*2)/ave_time/1000000000/2;
    printf("bandwidth is %f GB/s\n", bandwidth);
    return (0);
}