// Argo Beowulf Cluster: MPI Commands and Examples
// http://www.uic.edu/depts/accc/hardware/argo/mpi_routines.html

#include "mpi.h"
#include <stdio.h>
/* Derived datatype example program. Sending subset of entries in an array */
int main( int argc, char *argv[])
{
#define elements 4
        int numprocs, myrank, loop, count, block_length, stride;
        MPI_Datatype pointer_to_new_mpi_type;
        MPI_Status status;

        /* two dimensional array. Pass column 3 to other processes */
        int      x[elements][6]={
                            {10,20,30,40,50,60},
                            {70,80,90,100,110,120},
                            {130,140,150,160,170,180},
                            {190,200,210,220,230,240}
                          };

        int   y[elements] = {2,4,6,8};
        int   w[elements][6];
        int   sum = 0;

        MPI_Init(&argc,&argv);
        MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
        MPI_Comm_rank(MPI_COMM_WORLD, &myrank);

        if (myrank == 0) 
           printf("\nExample program to print noncontiguous items in a two-dimensional array\n\n");

        /* Build the derived datatype to pass column 3 of matrix x
           Pass x[0][3] containing 40
                x[1][3]     "      100
                x[2][3]     "      160
                x[3][3]     "      220

        count:  number of elements (# of array entries) to pass
        block_length: number of entries in each element
        stride:  number of elements between each element to pass
        */

        MPI_Bcast(y, elements, MPI_INT, 0, MPI_COMM_WORLD);
        count = elements;  // "elements" is how many rows of data there are
        block_length = 1;
        stride = 6;       // "stride" is the number of data items between
                          // consecutive column members
        MPI_Type_vector(count,block_length,stride,MPI_INT,
                                               &pointer_to_new_mpi_type);
        /* commit the new datatype */
        MPI_Type_commit(&pointer_to_new_mpi_type);

        /* Root contains the elements. Send them to other processes */
        if (myrank == 0) MPI_Send(&x[0][3],1,pointer_to_new_mpi_type,1,0,
                       MPI_COMM_WORLD);
        else {
            /* Non-root processes receive the variables and print them */
            MPI_Recv(&w[0][3], 1, pointer_to_new_mpi_type,0,0,MPI_COMM_WORLD,
                              &status);
            printf("In process with rank %d\n",myrank);
            for(loop=0;loop < elements;loop++){
                printf("\tw[%d][3]==> %d\n",loop,w[loop][3]);
            }
            printf("\n");
            printf("Array y:\n");
            for(loop=0; loop < elements; loop++) {
              printf("\t%d", y[loop]);   // backslash t means tab
            }
            printf("\n");
            for(loop=0; loop < elements; loop++) {
               sum = sum + y[loop] * w[loop][3];
            }
            printf("Dot product=%d\n", sum);
          }

        /* finish up */
        MPI_Type_free(&pointer_to_new_mpi_type);
        MPI_Finalize();
        return 0;
}
/*
cc vectormult.c -o /tmp/yourusername/a.out
Run this with: mpirun -np 2 /tmp/yourusername/a.out

The output should be:

Example program to print noncontiguous items in a two-dimensional array

In process with rank 1
        w[0][3]==> 40
        w[1][3]==> 100
        w[2][3]==> 160
        w[3][3]==> 220

Array y:
        2       4       6       8
Dot product=3200

*/