Step 1: create a PLI application program in C, such as, a "Hello world" as hereafter.

/**********************************************************************
 * $hello example -- PLI application using VPI routines
 *
 * C source to print "Hello World" as a PLI application.
 *
 * Usage: $hello;
 *
 * 
 *********************************************************************/

#include     /* ANSI C standard library */
#include      /* ANSI C standard input/output library */
#include   /* IEEE 1364 PLI VPI routine library  */

/**********************************************************************
 * calltf routine
 *********************************************************************/
PLI_INT32 hello_calltf(char *user_data)
{
  vpi_printf("\nHello World!\n\n");
  return(0);
}


/**********************************************************************
 * $hello Registration Data
 * (add this function name to the vlog_startup_routines array)
 *********************************************************************/
void hello_register()
{
  s_vpi_systf_data tf_data;

  tf_data.type        = vpiSysTask;
  tf_data.tfname      = "$hello";
  tf_data.calltf      = hello_calltf;
  tf_data.compiletf   = NULL;
  tf_data.sizetf      = NULL;
  vpi_register_systf(&tf_data);
}

#include 

/* prototypes of PLI application routine names */
int hello_calltf();

void hello_register()
{
  s_vpi_systf_data tf_data;

  tf_data.type = vpiSysTask;
  tf_data.sysfunctype =0;
  tf_data.tfname = "$hello";
  tf_data.calltf = hello_calltf;
  tf_data.compiletf = NULL;
  tf_data.sizetf = 0;
  tf_data.user_data = 0;
  vpi_register_systf(&tf_data);
}

In above $hello example, we do as follows:
g++ -c -I/[install_dir]/modeltech/include hello_vpi.c
g++ -c -I/[install_dir]/modeltech/include hello_register.c
ld -G -B symbolic -o mti_pli_apps.so hello_vpi.o hello_register.o

Step 4: Modify the ModelSim .ini file at /[install_dir]/modeltech/modelsim.ini
specify the names of the shared object libraries in the Veriuser variable. This variable is typically found towards the middle of the modelsim.ini file, and by default it is commented out. Any number of shared object files may be listed, separated by a white space. As an example, the $hello system task has shared object mti_pli_apps.so, then the Verisuer variable would be set to:

;List of dynamically loaded objects for Verilog PLI applications
Veriuser=mti_pli_apps.so

hello.v 

module hello(clk, reset);
input clk, reset;
initial
  begin
     $hello();
  end

endmodule

File test.do: 
#this is to run hello in cmd mode
do test_stim.do
quit -f


File test_stim.do

# setup an oscillator on the CLK input
force clk 1 50  -r 100
force clk 0 100 -r 100

# reset the clock and then count to 100
force reset 1
run 100

force reset 0
run 10000

Type following command in Dos/Unix prompt
$vsim -c -do test.do test -wlf test.wlf

The -c argument instructs ModelSim not to invoke the GUI. 
The -wlf argument saves the simulation results in a WLF file. 
This allows you to view the simulation results in the GUI for debugging purposes.