GLSL Shaders

Introduction

GLSL Shaders, shipped with PyMOL, may be modified to provide interesting and useful visual effects. The following modifications provide "Toon" shading effects, in some ways similar to the illustrations by David Goodsell as seen in the RCSB/PDB "Molecule of the Month" series [1].

Gallery

GLSL Toon Spheres

GLSL Toon Cartoon

Instructions

The fragment shaders have been modified for Cartoon and Spheres only as of now. The PyMOL shaders directory is located under:

• "/Applications/MacPyMOL.app/pymol/data/shaders" on Mac OSX

GLSL Toon Spheres

Sphere Fragment Shader Code

Rename the "sphere.fs" file in your PyMOL shaders directory to "sphere.fs.bak" then copy and save the following into a new file called "sphere.fs" (using a text editor) in the same directory:

// Sphere impostor fragment shader

// Sphere Toon Shading code modifications by Shivender Shandilya, 2012

uniform bool lighting_enabled;

uniform float ortho;

uniform float fog_enabled;
uniform bool bg_gradient;
uniform vec3 fog_color_top;
uniform vec3 fog_color_bottom;
uniform float inv_height;
uniform int light_count;
uniform float shininess;
uniform float shininess_0;
uniform int spec_count;
uniform float spec_value;
uniform float spec_value_0;
uniform int stereo_flag;
uniform mat3 matL;
uniform mat3 matR;
uniform float gamma;

varying vec4 COLOR;
varying vec3 sphere_center;
varying float radius2;
varying vec3 point;
// varying fog;

vec4 ComputeColorForLight(vec3 N, vec3 L, vec3 H, vec4 ambient, vec4 diffuse, float spec, float shine){
  float NdotL, NdotH;
  vec4 ret_val = vec4(0.);

  //
  // Begin code for calculating the Sphere Outline
  //
  float spheredge = dot(N,L);
  float edgeweight = 1.025;
  float edgegain = 16.0;
  spheredge = edgeweight - (spheredge * spheredge);
  spheredge = 1.0 - pow(spheredge, shine * edgegain);
  diffuse = mix(diffuse, diffuse * spheredge, shine);

  ret_val += diffuse * COLOR;

  NdotL = dot(N, L);
  if (NdotL > 0.0) {
    ret_val += diffuse * NdotL * COLOR;
    NdotH = max(dot(N, H), 0.0);
    ret_val += spec * pow(NdotH, shine);
  }
  return ret_val;
}

void main(void)
{

    vec3 ray_origin = mix(vec3(0.,0.,0.), point, ortho);

    vec3 ray_direction = mix(normalize(point), vec3(0., 0., 1.), ortho);

    vec3 sphere_direction = mix(sphere_center, ray_origin - sphere_center, ortho);

    // Calculate sphere-ray intersection
    float b = dot(sphere_direction, ray_direction);

    float position = b * b + radius2 - dot(sphere_direction, sphere_direction);

    // Check if the ray missed the sphere
    if (position < 0.0)
       discard;

    // Calculate nearest point of intersection
    float nearest = mix(b - sqrt(position), sqrt(position) - b, ortho);

    // Calculate intersection point on the sphere surface.  The ray
    // origin is at the quad (center point), so we need to project
    // back towards the user to get the front face.
    vec3 ipoint = nearest * ray_direction + ray_origin;

    // Calculate normal at the intersection point
    vec3 N = normalize(ipoint - sphere_center);

    // Calculate depth in clipping space
    vec2 clipZW = ipoint.z * gl_ProjectionMatrix[2].zw +
        gl_ProjectionMatrix[3].zw;

    float depth = 0.5 + 0.5 * clipZW.x / clipZW.y;

    // this is a workaround necessary for Mac
    // otherwise the modified fragment wont clip properly

    if (depth <= 0.0)
      discard;

    if (depth >= 1.0)
      discard;

    if (COLOR.a <= 1.0)
      gl_FragDepth = depth;
    else
      gl_FragDepth = 1.0;

    vec4 color;

    vec3 L0 = normalize(vec3(gl_LightSource[0].position) - ipoint);

    float NdotL = max(dot(N, L0), 0.0);

    float NdotH;

    vec4 final_color = gl_LightModel.ambient * COLOR;

/*
    if (light_count>0){
      final_color += ComputeColorForLight(N, normalize(vec3(gl_LightSource[0].position)),
                                          normalize(vec3(gl_LightSource[0].halfVector.xyz)),
                                          gl_LightSource[0].ambient,
                                          gl_LightSource[0].diffuse,
                                          spec_value_0, shininess_0);
      if (light_count>1){
        final_color += ComputeColorForLight(N, normalize(vec3(gl_LightSource[1].position)),
                                            normalize(vec3(gl_LightSource[1].halfVector.xyz)),
                                            gl_LightSource[1].ambient,
                                            gl_LightSource[1].diffuse,
                                            spec_value, shininess);
      if (light_count>2){
        final_color += ComputeColorForLight(N, normalize(vec3(gl_LightSource[2].position)),
                                            normalize(vec3(gl_LightSource[2].halfVector.xyz)),
                                            gl_LightSource[2].ambient,
                                            gl_LightSource[2].diffuse,
                                            spec_value, shininess);
      if (light_count>3){
        final_color += ComputeColorForLight(N, normalize(vec3(gl_LightSource[3].position)),
                                            normalize(vec3(gl_LightSource[3].halfVector.xyz)),
                                            gl_LightSource[3].ambient,
                                            gl_LightSource[3].diffuse,
                                            spec_value, shininess);
      if (light_count>4){
        final_color += ComputeColorForLight(N, normalize(vec3(gl_LightSource[4].position)),
                                            normalize(vec3(gl_LightSource[4].halfVector.xyz)),
                                            gl_LightSource[4].ambient,
                                            gl_LightSource[4].diffuse,
                                            spec_value, shininess);
      if (light_count>5){
        final_color += ComputeColorForLight(N, normalize(vec3(gl_LightSource[5].position)),
                                            normalize(vec3(gl_LightSource[5].halfVector.xyz)),
                                            gl_LightSource[5].ambient,
                                            gl_LightSource[5].diffuse,
                                            spec_value, shininess);
      if (light_count>6){
        final_color += ComputeColorForLight(N, normalize(vec3(gl_LightSource[6].position)),
                                            normalize(vec3(gl_LightSource[6].halfVector.xyz)),
                                            gl_LightSource[6].ambient,
                                            gl_LightSource[6].diffuse,
                                            spec_value, shininess);
      if (light_count>7){
        final_color += ComputeColorForLight(N, normalize(vec3(gl_LightSource[7].position)),
                                            normalize(vec3(gl_LightSource[7].halfVector.xyz)),
                                            gl_LightSource[7].ambient,
                                            gl_LightSource[7].diffuse,
                                            spec_value, shininess);
    }}}}}}}}
*/


    int i;
    for (i=0; i<light_count; i++){
      vec3 L = normalize(gl_LightSource[i].position.xyz);
      vec3 H = normalize(gl_LightSource[i].halfVector.xyz);
      float spec = 0., shine = 0.;
      if (i==0){
        spec = spec_value_0;
        shine = shininess_0;
      } else if (spec_count >= i){
        spec = spec_value;
        shine = shininess;
      }

      //      final_color += gl_LightSource[i].ambient * COLOR;

      final_color += ComputeColorForLight(N, normalize(vec3(gl_LightSource[i].position)),
                                          normalize(vec3(gl_LightSource[i].halfVector.xyz)),
                                          gl_LightSource[i].ambient * COLOR,
                                          gl_LightSource[i].diffuse * COLOR,
                                          spec, shine);

      NdotL = dot(N, L);

      if (NdotL > 0.0) {

        //    final_color += gl_LightSource[i].diffuse * NdotL * COLOR;

        final_color += ComputeColorForLight(N, normalize(vec3(gl_LightSource[i].position)),
                                            normalize(vec3(gl_LightSource[i].halfVector.xyz)),
                                            gl_LightSource[i].ambient * COLOR,
                                            gl_LightSource[i].diffuse * COLOR * NdotL * 0.333,
                                            spec, shine);
      }
        NdotH = max(dot(N, H), 0.0);
        final_color += spec * pow(NdotH, shine);
    }


    float fog = clamp((gl_Fog.end + ipoint.z) * gl_Fog.scale, 0.0, 1.0);
    fog = mix(1.0, fog, fog_enabled);
    vec3 fog_color;

    if (bg_gradient){
      fog_color = mix(fog_color_bottom, fog_color_top, gl_FragCoord.y * inv_height);
    } else {
      fog_color = fog_color_top;
    }

    final_color.rgb = mix(fog_color, final_color.rgb, fog);

    vec4 f = vec4(final_color.rgb, COLOR.a);

    //
    // Fake shadows based on depth
    //
    // vec4 f = vec4(final_color.rgb * ( 1.05 - gl_FragDepth), COLOR.a);

  if(stereo_flag==-1)
    gl_FragColor = vec4(matL * pow(f.rgb,vec3(gamma,gamma,gamma)), f.a);
  else if (stereo_flag==0)
    gl_FragColor = f;
  else if (stereo_flag==1)
    gl_FragColor = vec4(matR * pow(f.rgb, vec3(gamma,gamma,gamma)), f.a);
}

Sphere Testing PyMOL Script

Now test out the sphere shader by saving the following PyMOL script as "test1.pml" and running it from the "File > Run" menu:
(If File > Run does not work for you, try typing in "@test1.pml" on the PyMOL command line. First make sure to "cd" on the PyMOL command line to the directory where you saved test1.pml)

fetch 3V4K, async=0
show sphere, all
select solv, solvent
remove solv
delete solv
remove e. H
set sphere_mode, 9
set ambient, 0.25
set_color carbons, [100, 225,90]
set_color others, [75, 100, 230]
util.cbaw 3V4K
set sphere_scale, 1.05
set orthoscopic, 1

set_view (\
    -0.327029049,   -0.944887519,    0.015455140,\
     0.943798959,   -0.327394068,   -0.045230716,\
     0.047797613,   -0.000204348,    0.998857021,\
     0.000143192,   -0.000024423, -184.973327637,\
   -14.873787880,  -17.664234161,   35.063804626,\
   136.028396606,  233.926559448,   20.000000000 )

cmd.set('light_count',9)
cmd.set("light" ,"[-0.2,-0.2,-1.0]")
cmd.set("light2","[-0.2, 0.0,-1.0]")
cmd.set("light3","[-0.2, 0.2,-1.0]")
cmd.set("light4","[ 0.0, 0.2,-1.0]")
cmd.set("light5","[ 0.2, 0.2,-1.0]")
cmd.set("light6","[ 0.2, 0.0,-1.0]")
cmd.set("light7","[ 0.2,-0.2,-1.0]")
cmd.set("light8","[ 0.0,-0.2,-1.0]")
cmd.set("light9","[ 0.0,-0.2,-1.0]")

cmd.set('ambient',0.15)
cmd.set('direct',0.10)
cmd.set('reflect',0.80)
cmd.set('shininess',10)
cmd.set('spec_count',-1)
cmd.set('power',1.0)
cmd.set('specular_intensity',0)
cmd.set('spec_direct',0.0)
cmd.set('ray_shadow_decay_factor',0.1)
cmd.set('ray_shadow_decay_range',5.0)
cmd.set('bg_rgb',[1.0, 1.0, 1.0])


viewport 1024, 768
draw 1024, 768, 8

GLSL Toon Cartoons

Cartoon Fragment Shader Code

Rename the "default_es2.fs" file in your PyMOL shaders directory to "default_es2.fs.bak" then copy and save the following into a new file called "default_es2.fs" (using a text editor) in the same directory:

 
// Cartoon Toon Shading code modifications by Shivender Shandilya, 2012

varying vec3 packed_data_0 ;
varying vec4 packed_data_1 ;
varying vec4 packed_data_2 ;
varying vec4 packed_data_3 ;
varying vec4 packed_data_4 ;

//varying vec3 N;
#define NORMAL packed_data_0.xyz
#define COLOR packed_data_3
#define fog (packed_data_1.w )
#define fog_color packed_data_1.x, packed_data_1.y, packed_data_1.z

uniform float fog_enabled;

uniform bool lighting_enabled;
uniform bool two_sided_lighting_enabled;
uniform bool bg_gradient;
uniform int light_count;
uniform vec4 interior_color;
uniform float interior_color_threshold;
uniform float shininess;
uniform float shininess_0;
uniform bool use_interior_color_threshold;
uniform int spec_count;
uniform float spec_value;
uniform float spec_value_0;
uniform int stereo_flag;
uniform mat3 matR;
uniform mat3 matL;
uniform float gamma;

vec4 ComputeColorForLight(bool is_interior, bool two_sided_lighting_enabled,
                          vec3 L, vec3 H, vec4 ambient, vec4 diffuse, float spec, float shine){
  float NdotL, NdotH;
  vec4 ret_val = vec4(0.);

  if (!is_interior){

    //
    // Begin code for calculating the Cartoon Outline
    //
    float cartedge = dot(NORMAL, L);
    float edgeweight = 1.15;
    float edgegain = 16.0;
    cartedge = edgeweight - (cartedge * cartedge);
    cartedge = 1.0 - pow(cartedge, shine * edgegain);
    diffuse = mix(diffuse, diffuse * cartedge, shine);

    ret_val += diffuse * COLOR;

    NdotL = dot(NORMAL, L);

    if (NdotL > 0.0) {
       ret_val += diffuse * NdotL * COLOR;
       NdotH = max(dot(NORMAL, H), 0.0);
       ret_val += spec * pow(NdotH, shine);
    }
  }

  if (two_sided_lighting_enabled && is_interior){
    //
    // Begin code for calculating the Cartoon Outline
    //
    float cartedge = dot(NORMAL, L);
    float edgeweight = 1.15;
    float edgegain = 16.0;
    cartedge = edgeweight - (cartedge * cartedge);
    cartedge = 1.0 - pow(cartedge, shine * edgegain);
    diffuse = mix(diffuse, diffuse * cartedge, shine);

    ret_val += diffuse * COLOR;

    NdotL = dot(-NORMAL, L);

    if (NdotL > 0.0) {
       ret_val += diffuse * NdotL * COLOR;
       NdotH = max(dot(-NORMAL, H), 0.0);
       ret_val += spec * pow(NdotH, shine);
    }
  }

  return ret_val;
}

void main()
{
  vec4 final_color = vec4(0.);

  if (lighting_enabled){
    bool is_interior = false;
    if (use_interior_color_threshold){
      vec3 viewV = vec3(0.,0.,-1.);
      float dotp = dot(NORMAL, viewV);
      is_interior = ( dotp > interior_color_threshold );
    }
    if (!two_sided_lighting_enabled && is_interior){
      final_color = interior_color;
    } else {
      final_color = (gl_LightModel.ambient) * COLOR;

      if (light_count>0){
        final_color += ComputeColorForLight(is_interior, two_sided_lighting_enabled,
                                            normalize(vec3(gl_LightSource[0].position)),
                                            normalize(vec3(gl_LightSource[0].halfVector.xyz)),
                                            gl_LightSource[0].ambient,
                                            gl_LightSource[0].diffuse,
                                            spec_value_0, shininess_0);
      if (light_count>1){
        final_color += ComputeColorForLight(is_interior, two_sided_lighting_enabled,
                                            normalize(vec3(gl_LightSource[1].position)),
                                            normalize(vec3(gl_LightSource[1].halfVector.xyz)),
                                            gl_LightSource[1].ambient,
                                            gl_LightSource[1].diffuse,
                                            spec_value, shininess);
      if (light_count>2){
        final_color += ComputeColorForLight(is_interior, two_sided_lighting_enabled,
                                            normalize(vec3(gl_LightSource[2].position)),
                                            normalize(vec3(gl_LightSource[2].halfVector.xyz)),
                                            gl_LightSource[2].ambient,
                                            gl_LightSource[2].diffuse,
                                            spec_value, shininess);
      if (light_count>3){
        final_color += ComputeColorForLight(is_interior, two_sided_lighting_enabled,
                                            normalize(vec3(gl_LightSource[3].position)),
                                            normalize(vec3(gl_LightSource[3].halfVector.xyz)),
                                            gl_LightSource[3].ambient,
                                            gl_LightSource[3].diffuse,
                                            spec_value, shininess);
      if (light_count>4){
        final_color += ComputeColorForLight(is_interior, two_sided_lighting_enabled,
                                            normalize(vec3(gl_LightSource[4].position)),
                                            normalize(vec3(gl_LightSource[4].halfVector.xyz)),
                                            gl_LightSource[4].ambient,
                                            gl_LightSource[4].diffuse,
                                            spec_value, shininess);
      if (light_count>5){
        final_color += ComputeColorForLight(is_interior, two_sided_lighting_enabled,
                                            normalize(vec3(gl_LightSource[5].position)),
                                            normalize(vec3(gl_LightSource[5].halfVector.xyz)),
                                            gl_LightSource[5].ambient,
                                            gl_LightSource[5].diffuse,
                                            spec_value, shininess);
      if (light_count>6){
        final_color += ComputeColorForLight(is_interior, two_sided_lighting_enabled,
                                            normalize(vec3(gl_LightSource[6].position)),
                                            normalize(vec3(gl_LightSource[6].halfVector.xyz)),
                                            gl_LightSource[6].ambient,
                                            gl_LightSource[6].diffuse,
                                            spec_value, shininess);
      if (light_count>7){
        final_color += ComputeColorForLight(is_interior, two_sided_lighting_enabled,
                                            normalize(vec3(gl_LightSource[7].position)),
                                            normalize(vec3(gl_LightSource[7].halfVector.xyz)),
                                            gl_LightSource[7].ambient,
                                            gl_LightSource[7].diffuse,
                                            spec_value, shininess);
      }}}}}}}}
    }
/*
      for (i=0; i<light_count;i++){
        vec3 L = normalize(vec3(gl_LightSource[i].position));
        vec3 H = normalize(vec3(gl_LightSource[i].halfVector.xyz));
        vec4 ambient = gl_LightSource[i].ambient;
        vec4 diffuse = gl_LightSource[i].diffuse;
        float spec = 0., shine = 0.;
        if (i==0){
          spec = spec_value_0;
          shine = shininess_0;
        } else if (spec_count >= i){
          spec = spec_value;
          shine = shininess;
        }
        final_color += ComputeColorForLight(is_interior, two_sided_lighting_enabled, L, H, ambient, diffuse, spec, shine);
      }

    }
*/
  } else {
    final_color = COLOR;
  }

  float cfog = mix(1.0, clamp(fog, 0.0, 1.0), fog_enabled);

  vec4 f = vec4(mix(vec3(fog_color), final_color.rgb, cfog), COLOR.a);

  if(stereo_flag==-1)
    gl_FragColor = vec4(matL * pow(f.rgb,vec3(gamma,gamma,gamma)), f.a);
  else if (stereo_flag==0)
    gl_FragColor = f;
  else if (stereo_flag==1)
    gl_FragColor = vec4(matR * pow(f.rgb, vec3(gamma,gamma,gamma)), f.a);
}

Cartoon Testing PyMOL Script

Now test out the sphere shader by saving the following PyMOL script as "test2.pml" and running it from the "File > Run" menu:
(If File > Run does not work for you, try typing in "@test2.pml" on the PyMOL command line. First make sure to "cd" on the PyMOL command line to the directory where you saved test2.pml)

fetch 3V4K, async=0
hide all
show cartoon, all
select solv, solvent
remove solv
delete solv
remove e. H
set ambient, 0.25
color brown, 3V4K
set orthoscopic, 1

set cartoon_dumbbell_length, 0.85
set cartoon_dumbbell_radius, 0.15
set cartoon_dumbbell_width, 0.15
set cartoon_fancy_helices, on

set cartoon_rect_length, 0.85
set cartoon_rect_width, 0.15

set cartoon_loop_radius, 0.15

set_view (\
    -0.811935484,    0.547164500,   -0.203404382,\
     0.385375530,    0.240704626,   -0.890811384,\
    -0.438461006,   -0.801668167,   -0.406299770,\
     0.000018395,    0.000004439,  -80.686973572,\
   -14.601868629,   -7.145599365,   10.938076973,\
    26.767528534,  134.606826782,   20.000000000 )

cmd.set('light_count',9)
cmd.set("light" ,"[0.9,0.9,-1.0]")
cmd.set("light2","[0.9,0.9,-1.0]")
cmd.set("light3","[0.9,0.9,-1.0]")
cmd.set("light4","[0.9,0.9,-1.0]")
cmd.set("light5","[0.9,0.9,-1.0]")
cmd.set("light6","[0.9,0.9,-1.0]")
cmd.set("light7","[0.9,0.9,-1.0]")
cmd.set("light8","[0.9,0.9,-1.0]")
cmd.set("light9","[0.9,0.9,-1.0]")

cmd.set('ambient',0.15)
cmd.set('direct',0.10)
cmd.set('reflect',0.80)
cmd.set('shininess',0)
cmd.set('spec_count',0)
cmd.set('power',1.0)
cmd.set('specular_intensity',0)
cmd.set('spec_direct',0.0)
cmd.set('ray_shadow_decay_factor',0.1)
cmd.set('ray_shadow_decay_range',5.0)
cmd.set('bg_rgb',[0.75, 0.75, 0.75])


viewport 1024, 768
draw 1024, 768, 8

Epilogue

The above is just a (very hackish) start. Feel free to modify, test and edit this Wiki page with the changes/improvements you make. Have fun!