Q:

How to write the mexFunction of this c file

The function is cyclic.c.
  void cyclic(float a[], float b[], float c[], float alpha, float beta,
    float r[], float x[], unsigned long n)

   // Solves for a vector x[1..n] the “cyclic” set of linear equations. a,
    //b, c, and r are input vectors, all dimensioned as [1..n], while alpha and beta are        //the corner
  //  entries in the matrix.

I am new for the interface between Matlab and C. And I have not use C for several years. 
Last night, I finished it and compile. The last thing is to call it.
#include "mex.h" 

#include "nrutil.h"

#define FREE_ARG char*
#define NR_END 1

#include <stdio.h>
#include <stddef.h>
#include <stdlib.h>
#define NR_END 1
#define FREE_ARG char*
void nrerror(char error_text[])
/* Numerical Recipes standard error handler */
{fprintf(stderr,"Numerical Recipes run-time error...\n");
fprintf(stderr,"%s\n",error_text);
fprintf(stderr,"...now exiting to system...\n");
exit(1);
}
float *vector(long nl, long nh)
/* allocate a float vector with subscript range v[nl..nh] */
{
float *v;
v=(float *)malloc((size_t) ((nh-nl+1+NR_END)*sizeof(float)));
if (!v) nrerror("allocation failure in vector()");
return v-nl+NR_END;
}

void free_vector(float *v, long nl, long nh)
/* free a float vector allocated with vector() */
{
free((FREE_ARG) (v+nl-NR_END));
}

void tridag(float a[], float b[], float c[], float r[], float u[],
unsigned long n)
{
unsigned long j;
float bet,*gam;
gam=vector(1,n); 
if (b[1] == 0.0) nrerror("Error 1 in tridag");

u[1]=r[1]/(bet=b[1]);
for (j=2;j<=n;j++) {
gam[j]=c[j-1]/bet;
bet=b[j]-a[j]*gam[j];
if (bet == 0.0) nrerror("Error 2 in tridag"); 
        u[j]=(r[j]-a[j]*u[j-1])/bet; 
}
for (j=(n-1);j>=1;j--)
u[j] -= gam[j+1]*u[j+1]; 
free_vector(gam,1,n);
}

void cyclic(float a[], float b[], float c[], float alpha, float beta,
float r[], float x[], unsigned long n)

{
void tridag(float a[], float b[], float c[], float r[], float u[],
unsigned long n);
unsigned long i;
float fact,gamma,*bb,*u,*z;
if (n <= 2) nrerror("n too small in cyclic");
bb=vector(1,n);
u=vector(1,n);
z=vector(1,n);
gamma = -b[1]; //Avoid subtraction error in forming bb[1].
bb[1]=b[1]-gamma; //Set up the diagonal of the modified tridiagonal
bb[n]=b[n]-alpha*beta/gamma; //system.
for (i=2;i<n;i++) bb[i]=b[i];
tridag(a,bb,c,r,x,n);// Solve A · x = r.
u[1]=gamma;// Set up the vector u.
u[n]=alpha;
for (i=2;i<n;i++) u[i]=0.0;
tridag(a,bb,c,u,z,n);// Solve A · z = u.
fact=(x[1]+beta*x[n]/gamma)/ //Form v · x/(1 + v · z).
(1.0+z[1]+beta*z[n]/gamma);
for (i=1;i<=n;i++) x[i] -= fact*z[i]; //Nowget the solution vector x.
free_vector(z,1,n);
free_vector(u,1,n);
free_vector(bb,1,n);
}

void mexFunction(int nlhs, mxArray *plhs[], 
        int nrhs, const mxArray *prhs[]) 
    { 

   float *a,*b,*c,*x,*r;
   float alpha,beta;
unsigned long n = (unsigned long) mxGetScalar(prhs[6]);
 //   a=mxGetPr(prhs[0]); 
  //  b=mxGetPr(prhs[1]); 
  //  c=mxGetPr(prhs[2]);
  //  r=mxGetPr(prhs[5]);
    a = (float*) mxGetData(prhs[0]);
    b = (float*) mxGetData(prhs[1]);
    c = (float*) mxGetData(prhs[2]);
    r = (float*) mxGetData(prhs[5]);
   // alpha=*(mxGetPr(prhs[3]));
   // beta=*(mxGetPr(prhs[4]));
    alpha = (float) mxGetScalar(prhs[3]);
 beta = (float) mxGetScalar(prhs[4]);

    plhs[0]= mxCreateDoubleMatrix(n, 1, mxREAL);
    x = mxGetPr(plhs[0]);
    mexPrintf("%f  ",alpha); 
    mexPrintf("\n"); 
    mexPrintf("%f  ",beta); 
    mexPrintf("\n"); 
    mexPrintf("%d  ",n);
    mexPrintf("\n"); 

    cyclic(a,b,c, alpha, beta,r,x,n) ;
    mexPrintf("%d  ",n);
    mexPrintf("\n"); 
    } 

Finally I successfully compile itcyclic(a,b,c, alpha, beta,r,x,n) ;. But the answer is not right. I thing this is because r is an imaginary vector. So my question is how should I transform r between C and Matlab?

A:

The C function cyclic expects arrays of floats, but mexFunction is passing a double*. Without changing cyclic.c, you have two options:

Convert the data to single in MATLAB and get a float* with mxGetData.
In mexFunction:
float *a = (float*) mxGetData(prhs[0]);

In MATLAB:
mexFunction(single(a),...)

Convert (copy, not cast!) the data in mexFunction.
In mexFunction, allocate new float arrays, and copy each element from the double input array (mxGetPr(prhs[0])) into the temporary float array.
Call mexFunction with a normal double array in MATLAB.

It's probably easier to do the former.
Under no circumstances should you simply cast the pointer, not that you were planning to do that.

Also, the scalars alpha, beta and n need to be read from prhs as scalars and passed to cyclic as scalars. In mexFunction, use:
float alpha = (float) mxGetScalar(prhs[...]);
float beta = (float) mxGetScalar(prhs[...]);
unsigned long n = (unsigned long) mxGetScalar(prhs[...]);

You've entirely forgotten c and r in mexFunction.

