source: GTP/trunk/App/Demos/Vis/FriendlyCulling/src/shaders/antialiasing.cg @ 2867

struct v2p
{
        float2 c:  TEXCOORD0; // center
        float2 lt: TEXCOORD1; // left top
        float2 rb: TEXCOORD2; // right bottom
        float2 rt: TEXCOORD3; // right top
        float2 lb: TEXCOORD4; // left bottom
        float4 lr: TEXCOORD5; // left / right
        float4 tb: TEXCOORD6; // top / bottom
};

// the barrier for detecting a discontinuity
uniform float4 e_barrier = float4(5e-3, 5e-3, 0, 0); // x = normal, y = depth
// the weights for normal / depth discontinuity
uniform float4 e_weights = float4(1.0f, 1.0f, 1.0f, 1.0f); // x = normal, y = depth
uniform float4 e_kernel = float4(0.35f, 1.0f, 1.0f, 1.0f); 


float4 main(v2p IN, 
                        uniform sampler2D colors,
                        uniform sampler2D normals
                        ): COLOR
{
        //return tex2D(colors, IN.c.xy);
        // normal discontinuity filter
        float3 nc = (float3)tex2D(normals, IN.c.xy);

        float4 nd;
        nd.x = dot(nc, float3(tex2D(normals, IN.lt.xy)));
        nd.y = dot(nc, float3(tex2D(normals, IN.rb.xy)));
        nd.z = dot(nc, float3(tex2D(normals, IN.rt.xy)));
        nd.w = dot(nc, float3(tex2D(normals, IN.lb.xy)));

        nd -= e_barrier.x;
        nd = step(0.0f, nd);

        float ne = saturate(dot(nd, e_weights.x));

        // opposite coordinates
        float4 lrr = IN.lr.wzyx;
        float4 tbr = IN.tb.wzyx;

        // depth filter: compute gradient difference:
        // (c - sample) + (c - opposite sample)

        float4 dc = float4(tex2D(colors, IN.c).w);

        float pos_lt = (float)tex2D(colors, IN.lt.xy).w;
        float pos_rb = (float)tex2D(colors, IN.rb.xy).w;

        float pos_lb = (float)tex2D(colors, IN.lb.xy).w;
        float pos_rt = (float)tex2D(colors, IN.rt.xy).w;

        float pos_l = (float)tex2D(colors, IN.lr.xy).w;
        float pos_r = (float)tex2D(colors, lrr.xy).w;

        float pos_t = (float)tex2D(colors, IN.tb.xy).w;
        float pos_b = (float)tex2D(colors, tbr.xy).w;

        float4 dd;

        dd.x = pos_lt + pos_rb;
        dd.y = pos_lb + pos_rt;
        dd.z = pos_l + pos_r;
        dd.w = pos_t + pos_b;

        dd = abs(2.0f * dc - dd) - e_barrier.y;
        dd = step(dd, 0.0f);

        float de = saturate(dot(dd, e_weights.y));

        // weight: 0 = no aa, 1 = full antialiasing
        float w = (1.0f - de * ne) * e_kernel.x;

        // smoothed color
        // (a - c) * w + c = a * w + c * (1 - w)
        float2 offset = IN.c.xy * (1.0f - w);

        float4 s0 = tex2D(colors, offset + IN.lt.xy * w);
        float4 s1 = tex2D(colors, offset + IN.rb.xy * w);
        float4 s2 = tex2D(colors, offset + IN.rt.xy * w);
        float4 s3 = tex2D(colors, offset + IN.lb.xy * w);

        return (s0 + s1 + s2 + s3) / 4.0f;
}