source: GTP/trunk/App/Demos/Illum/HierRayEngine/rayEngine.mine.fx @ 1481

// transformations
float4x4        worldProjection;
float4x4        screenToWorld;
float4x4        modelWorld;
float4x4        worldModel;
float3          eyePos;
float3          tilePos;
float           rf;     //refrac coeff
float           colOrder;
bool            fini;
#define maxRayDepth     200.0

float4 conePeak;
float4 coneDira;

texture rayDirTable;
sampler rayDirTableSampler = sampler_state
{
   Texture = <rayDirTable>;
   MinFilter = Point;
   MagFilter = Point;
   MipFilter = None;
};

texture rayOriginTable;
sampler rayOriginTableSampler = sampler_state
{
   Texture = <rayOriginTable>;
   MinFilter = Point;
   MagFilter = Point;
   MipFilter = None;
};

texture coneDirTable;
sampler coneDirTableSampler = sampler_state
{
   Texture = <coneDirTable>;
   MinFilter = Point;
   MagFilter = Point;
   MipFilter = None;
};

texture conePeakTable;
sampler conePeakTableSampler = sampler_state
{
   Texture = <conePeakTable>;
   MinFilter = Point;
   MagFilter = Point;
   MipFilter = None;
};

textureCUBE environmentTexture;
sampler environmentSampler = sampler_state
{
   Texture = <environmentTexture>;
   MinFilter = Linear;
   MagFilter = Linear;
   MipFilter = None;
};

struct vsInputRenderPrimaryRayArray
{
    float4      pos                     : POSITION;
    float3      normal          : NORMAL;
};

struct vsOutputRenderPrimaryRayArray
{
    float4      pos                     : POSITION;
    float3      normal          : TEXCOORD0;
    float3      worldPos        : TEXCOORD1;
};

vsOutputRenderPrimaryRayArray
        vsRenderPrimaryRayArray(vsInputRenderPrimaryRayArray input)
{
    vsOutputRenderPrimaryRayArray output = (vsOutputRenderPrimaryRayArray)0;

        output.pos = mul(input.pos, worldProjection);
        output.normal = input.normal;
        float4 worldPos = mul(input.pos, modelWorld);
        output.worldPos = worldPos.xyz / worldPos.w;

    return output;
}

struct psOutputRenderPrimaryRayArray
{
    float4      origin  : COLOR0;
    float4      dir             : COLOR1;
};

psOutputRenderPrimaryRayArray
        psRenderPrimaryRayArray(vsOutputRenderPrimaryRayArray input) : COLOR
{   
        psOutputRenderPrimaryRayArray output = (psOutputRenderPrimaryRayArray)0;
        output.origin = float4(input.worldPos, 1);
        float3 normal = normalize(input.normal);
        float3 viewDir =  input.worldPos - eyePos;
        viewDir = normalize(viewDir);

        float3 rfd = refract(viewDir, normal, rf);
        if(dot(rfd, rfd) < 0.5)
        {
                rfd = reflect(viewDir, normal);
        }
        output.dir = float4(rfd, colOrder);
        
        output.dir.xyz = normalize(output.dir.xyz);

        return output;
}

struct vsInputCopyBack
{
    float4      pos                     : POSITION;
    float2      tex                     : TEXCOORD0;    
};

struct vsOutputCopyBack
{
    float4      pos                     : POSITION;
    float2      tex                     : TEXCOORD0;
};

vsOutputCopyBack
        vsCopyBack(vsInputCopyBack input)
{
    vsOutputCopyBack output = (vsOutputCopyBack)0;
    output.pos = input.pos;
    output.tex = input.tex;
    return output;
}

void
        psCopyBack(vsOutputCopyBack input, out float4 c0 : COLOR0, out float4 c1 : COLOR1)
{
        c0 = tex2D(rayOriginTableSampler, input.tex);
        c1 = tex2D(rayDirTableSampler, input.tex);
}

struct vsInputShowTex
{
    float4      pos                     : POSITION;
    float2      tex                     : TEXCOORD0;    
};

struct vsOutputShowTex
{
    float4      pos                     : POSITION;
    float2      tex                     : TEXCOORD0;
};

vsOutputShowTex
        vsShowTex(vsInputShowTex input)
{
    vsOutputShowTex output = (vsOutputShowTex)0;
    output.pos = input.pos;
    output.tex = input.tex;

    return output;
}

float4
        psShowTex(vsOutputShowTex input) : COLOR0
{   
//      return abs(float4(tex2D(rayDirTableSampler, input.tex).xyz, 1));
        return float4(tex2D(rayOriginTableSampler, input.tex).xyz, 1);
//      return abs(float4(tex2D(conePeakTableSampler, input.tex).xyz, 1));      
//      return float4(1,1,0,1); 
//      return float4(input.tex.x,input.tex.y,0,1);     
//      return float4(1.0 - acos(tex2D(coneDirTableSampler, input.tex).aaa) / 3.14, 1); 
//       + float4(0, 0.7, 0, 0);
//              +       float4(tex2D(rayDirTableSampler, input.tex).xyz, 1);
        float3 relfDir = tex2D(rayDirTableSampler, input.tex);
        return float4((relfDir.x +1) / 2,
                (relfDir.y + 1) / 2,
                (relfDir.z + 1) / 2,   1);
}

struct vsInputRayCast
{
    half4       sphere          : POSITION;
    half3       planePos        : NORMAL;
    half3  invmx0               : TEXCOORD0;
    half3  invmx1               : TEXCOORD1;
    half3  invmx2               : TEXCOORD2;
    half3  normals0     : TEXCOORD3;
        half3  normals1 : TEXCOORD4;
        half3  normals2 : TEXCOORD5;
        half3  tex0             : TEXCOORD6;
        half3  tex1             : TEXCOORD7;
        half3  tex2             : TEXCOORD8;    
};

struct vsOutputRayCast
{
        half4   pos             : POSITION;
    half3  invmx0               : TEXCOORD0;
    half3  invmx1               : TEXCOORD1;
    half3  invmx2               : TEXCOORD2;
    half3  normals0     : TEXCOORD3;
        half3  normals1 : TEXCOORD4;
        half3  normals2 : TEXCOORD5;
        half3  tex0             : TEXCOORD6;
        half3  tex1             : TEXCOORD7;
//      float3  tex2            : TEXCOORD8;
    half3       planePos        : TEXCOORD8;
};

vsOutputRayCast
        vsRayCast(vsInputRayCast input)
{
    vsOutputRayCast output = (vsOutputRayCast)0;
        output.planePos = input.planePos;
        output.invmx0 = input.invmx0;
        output.invmx1 = input.invmx1;
        output.invmx2 = input.invmx2;
        output.normals0 = input.normals0;
        output.normals1 = input.normals1;
        output.normals2 = input.normals2;
        output.tex0 = input.tex0;
        output.tex1 = input.tex1;
//      output.tex2 = input.tex2;
        output.pos = float4(tilePos, 1);

//      float2 coneTexCoords = tilePos * 0.5 + float2(0.5 + 1.0/32.0, 0.5 + 1.0/32.0);
        tilePos.y = - tilePos.y;
//      float2 coneTexCoords = (tilePos + 1.0 - 1.0/32.0) * 0.5;
//      float4 conePeak = tex2Dlod(conePeakTableSampler, float4(coneTexCoords, 0, 0));
//      float4 coneDira = tex2Dlod(coneDirTableSampler, float4(coneTexCoords, 0, 0));

//      conePeak.xyz = mul(float4(conePeak.xyz, 1.0), worldModel);

        float3 sfc = input.sphere.xyz - conePeak.xyz;
        float lsfc = length(sfc);

        if(input.sphere.w < lsfc)
        {
                float angSpMidConeMid = acos(dot(coneDira.xyz, sfc) / lsfc);
                float angSpRad = asin(input.sphere.w / lsfc);
                float angCone = acos(coneDira.w);
                
                if(angCone + angSpRad < angSpMidConeMid)
                {
                        output.pos = float4(10000000.0, 10000000.0, 10000000.0, 1.0);
                }
        }/**/

    return output;
}

void
        psRayCast(vsOutputRayCast input, in float2 vpos : VPOS,  out float4 origin : COLOR0, out float4 dir : COLOR1, out float1 depth : DEPTH)
{   

        float2 pixpos = vpos.xy;
        pixpos /= 512.0;
        float4 rayOrigin = tex2D(rayOriginTableSampler, pixpos.xy);
        rayOrigin = mul(rayOrigin, worldModel);
        float4 rayDirDepth = tex2D(rayDirTableSampler, pixpos.xy);
        float3 rayDir = rayDirDepth.xyz;
        
        float hitDepth = (dot(input.planePos, input.planePos) - dot(rayOrigin, input.planePos)) / dot(rayDir, input.planePos);
        float toler = 0.1;
        if(hitDepth < toler || hitDepth > maxRayDepth)
        {       
                depth = 1000000.0;
                origin = dir = 0;
        }
        else
        {
                float3 hitPoint = rayOrigin + (rayDir * hitDepth);

                float4(10.0, 0, 0, 1);
                depth = hitDepth / maxRayDepth;

                float baryA = dot(input.invmx0, hitPoint);
                float baryB = dot(input.invmx1, hitPoint);
                float baryC = dot(input.invmx2, hitPoint);
                if(baryA > -0.001 && baryB > -0.001 && baryC > -0.001)
                {
                        origin = mul(float4(hitPoint, 1), modelWorld);
                        float3 normalAtHitPoint = input.normals0 * baryA;
                        normalAtHitPoint += input.normals1 * baryB;
                        normalAtHitPoint += input.normals2 * baryC;
                        normalAtHitPoint = normalize(normalAtHitPoint);
                        if(dot(normalAtHitPoint, rayDir) > 0)
                        {
                                normalAtHitPoint = -normalAtHitPoint;
                                rf = 1.0 / rf;
                        }
                        float3 rfd = refract(rayDir, normalAtHitPoint, rf);
                        if(dot(rfd, rfd) < 0.5)
                                rfd = reflect(rayDir, normalAtHitPoint);
                        dir = float4(rfd, rayDirDepth.w + colOrder);
//                      dir = float4(reflect(rayDir, normalAtHitPoint), 1);
                }
                else
                {
                        origin = float4(10.0, 0, 0, 1);
                        dir = float4(10.0, 0, 0, 1);
                        depth = 1000000.0;
                }
        }
}


struct vsInputBackGround
{
    float4 pos  : POSITION; //Pos
    float2 tex  : TEXCOORD0;
};

struct vsOutputBackGround
{
    float4 pos  : POSITION; //Pos
        float2 tex      : TEXCOORD0;
    float3 worldPos : TEXCOORD1;
};

vsOutputBackGround vsBackground(vsInputBackGround input){
    vsOutputBackGround output = (vsOutputBackGround)0;
    output.pos=input.pos;
    float4 worldpos = mul(input.pos, screenToWorld);
    worldpos /= worldpos.w;
    output.worldPos = worldpos.xyz;
    output.tex = input.tex;
    return output;
}

void psBackground(vsOutputBackGround input, out float4 o0 : COLOR0) 
{   
        float4 dird = tex2D(rayDirTableSampler, input.tex);
        float3 dir = dird.xyz;
        float3 col = 1 ;
                                                /*float3(pow(0.7, frac(dird.w / 10.0) * 10.0) ,
                                                pow(0.7, frac(dird.w / 100.0) * 10.0) ,
                                                pow(0.7, frac(dird.w / 1000.0) * 10.0));*/
        if(dot(dir, dir) < 0.5)
        {
                dir = normalize(input.worldPos.xyz - eyePos.xyz);
                col = float3(4, 4, 4);
        }
        o0 = float4(texCUBE(environmentSampler, dir).xyz * col * 1.1, 0.0);
//      o0 = float4(dird.w / 5.0, dird.w / 2.0, dird.w / 10.0, 1.0);
        
//      o0 = float4(input.tex.x, input.tex.y, 1.0, 1.0);
}

struct vsInputCone
{
    float4      pos                     : POSITION;
    float2      tex                     : TEXCOORD0;    
};

struct vsOutputCone
{
    float4      pos                     : POSITION;
    float2      tex                     : TEXCOORD0;
};

vsOutputCone
        vsCone(vsInputCone input)
{
    vsOutputCone output = (vsOutputCone)0;
    output.pos = input.pos;
    output.tex = input.tex;
    return output;
}

void
        psCone(vsInputCone input, out float4 rpeak : COLOR0, out float4 rdir : COLOR1)
{
//      rpeak = rdir = tex2D(rayDirTableSampler, input.tex + 2.0/32.0);
//      return;
        float3 peak = float3(0, 0, 0);//tex2D(rayOriginTableSampler, input.tex + 1.0/32.0);
        float3 dir = float3(0, 0, 0);//tex2D(rayDirTableSampler, input.tex + 1.0/32.0);
        float cosAngle = 1.0;
        float sinAngle = 0.0;
        for(int i=0; i < 16; i++)
                for(int j=0; j < 16; j++)
                {
                        float3 ndir = tex2D(rayDirTableSampler, input.tex + float2((float)j/512.0, (float)i/512.0));
                        if(dot(ndir, ndir) > 0.5)
                        {
                                float3 npeak = tex2D(rayOriginTableSampler, input.tex + float2((float)j/512.0, (float)i/512.0));
                                if(dot(dir,dir) < 0.5)
                                {
                                        dir = ndir;
                                        peak = npeak;
                                }
                                else
                                {
                                        float ncos = dot(ndir,dir);
                                        if(ncos < cosAngle)
                                        {
                                                float3 perpdir = normalize(ndir - ncos * dir);
                                                float3 farConeEdge = cosAngle * dir - sinAngle * perpdir;
                                                dir = farConeEdge + ndir;
                                                dir = normalize(dir);
                                                cosAngle = dot(dir, ndir);
                                                sinAngle = sqrt(1.0 - cosAngle * cosAngle);
                                        }
                                        float3 pd = npeak - peak;
                                        float3 ptop = normalize(pd);
                                        float cospp = dot(dir, ptop);
                                        if(cospp < cosAngle)
                                        {
                                                float3 perpdir = ptop - cospp * dir;
                                                perpdir = normalize(perpdir);
                                                float3 farConeEdge = (cosAngle * dir - sinAngle * perpdir);
                                                float3 nearConeEdge = (cosAngle * dir + sinAngle * perpdir);
                                                float3 g = npeak - peak - nearConeEdge * dot(nearConeEdge, pd);
                                                peak += farConeEdge * dot(g,g) / dot(farConeEdge, g);
                                        }
                                }
                        }
                }
        rpeak = float4(peak, 1.0);
        rdir = float4(dir, cosAngle);
        if(dot(dir, dir) < 0.5)
        {
                rpeak = float4(1000000.0, 0 , 0, 0.0);
                rdir = float4(1, 0, 0, 1.0);
        }
}

struct vsInputCheck
{
    float4      pos                     : POSITION;
    float2      tex                     : TEXCOORD0;    
};

struct vsOutputCheck
{
    float4      pos                     : POSITION;
    float2      tex                     : TEXCOORD0;
};

vsOutputCheck
        vsCheck(vsInputCheck input)
{
    vsOutputCone output = (vsOutputCone)0;
    output.pos = input.pos;
    output.tex = input.tex;
    return output;
}

float4 psCheck(vsOutputCheck input) : COLOR
{
        float3 orig = tex2D(rayOriginTableSampler, input.tex);
        float3 dir = tex2D(rayDirTableSampler, input.tex);
        float3 peak = tex2D(conePeakTableSampler, input.tex);
        float4 coneDira = tex2D(coneDirTableSampler, input.tex);        
        float4 ret = float4(0, 1, 0, 1);;
        if(dot(dir, coneDira.xyz) < coneDira.a - 0.01)
                ret = float4(1, 0, 0, 1);
        if( (dot(normalize(orig - peak), coneDira.xyz) < coneDira.a - 0.01))
                ret = float4(0, 0, 1, 1);
//      ret = float4(dir + coneDira.xyz, 1);
//      return abs(float4(peak, 1))  * 100.0;
        return ret;
}

technique RenderPrimaryRayArray{
        pass P0
    {
        VertexShader = compile vs_3_0 vsRenderPrimaryRayArray();
        PixelShader  = compile ps_3_0 psRenderPrimaryRayArray();
    }
}

technique RayCast{
        pass P0
    {
        VertexShader = compile vs_3_0 vsRayCast();
        PixelShader  = compile ps_3_0 psRayCast();
    }
}

technique ShowTex{
        pass P0
    {
        VertexShader = compile vs_2_0 vsShowTex();
        PixelShader  = compile ps_2_0 psShowTex();
    }
}

technique Background{
        pass P0
    {
        VertexShader = compile vs_2_0 vsBackground();
        PixelShader  = compile ps_2_0 psBackground();
    }
}

technique CopyBack{
        pass P0
    {
        VertexShader = compile vs_2_0 vsCopyBack();
        PixelShader  = compile ps_2_0 psCopyBack();
    }
}

technique ComputeCones{
        pass P0
    {
        VertexShader = compile vs_3_0 vsCone();
        PixelShader  = compile ps_3_0 psCone();
    }
}

technique ConeChecker{
        pass P0
    {
        VertexShader = compile vs_3_0 vsCheck();
        PixelShader  = compile ps_3_0 psCheck();
    }
}