UE4：Shader Permutation & Compile

如需转载本文，请声明作者及出处。

新增Shader时，通常需要用到以下的宏

DECLARE_SHADER_TYPE(FMobileXXXVS, MeshMaterial);
DECLARE_SHADER_TYPE(FMobileXXXPS, MeshMaterial);
IMPLEMENT_MATERIAL_SHADER_TYPE(, FMobileXXXVS, TEXT("/Engine/Private/XXXPassShader.usf"), TEXT("MainVS"), SF_Vertex);
IMPLEMENT_MATERIAL_SHADER_TYPE(, FMobileXXXPS, TEXT("/Engine/Private/XXXPassShader.usf"), TEXT("MainPS"), SF_Pixel);
IMPLEMENT_VERTEX_FACTORY_TYPE(FXXXVertexFactory, "/Engine/Private/XXXPassShader.usf", true, false, false, false, false);

才能实现c++类与Shader的绑定，才能在MeshPipeline中，AddMeshBatch时取到相应的shader的。
但在ue4中，是如何做到的呢？

Shader Permutation

1.几个关键宏的展开

先来看看DECLARE_SHADER_TYPE的展开

#define DECLARE_EXPORTED_SHADER_TYPE(ShaderClass,ShaderMetaTypeShortcut,RequiredAPI, ...) \
    public: \
    using FPermutationDomain = FShaderPermutationNone; \
    using ShaderMetaType = F##ShaderMetaTypeShortcut##ShaderType; \
    \
    static RequiredAPI ShaderMetaType StaticType; \
    \
    static FShader* ConstructSerializedInstance() { return new ShaderClass(); } \
    static FShader* ConstructCompiledInstance(const ShaderMetaType::CompiledShaderInitializerType& Initializer) \
    { return new ShaderClass(Initializer); } \
    \
    virtual uint32 GetTypeSize() const override { return sizeof(*this); }

以TMobileBasePassPS为例

template< typename LightMapPolicyType, EOutputFormat OutputFormat, bool bEnableSkyLight, bool bRenderRuntimeProceduralTexture, int32 NumMovablePointLights>
class TMobileBasePassPS : public TMobileBasePassPSBaseType
{
    DECLARE_SHADER_TYPE(TMobileBasePassPS,MeshMaterial);
}

会被展开为：

class TMobileBasePassPS : public TMobileBasePassPSBaseType
{
using ShaderMetaType = FMeshMaterialShaderType;
static RequiredAPI ShaderMetaType StaticType;
}

如此，TMobileBasePassPS便有了ShaderMetaType 和 ShaderMetaType

而FShaderType是模板类的实例化类的静态成员，FShaderType构造函数里面会把自己注册到全局链表FShaderType::GetTypeList()里：

TLinkedList*& FShaderType::GetTypeList()
{
    return GShaderTypeList;
}

FShaderType::FShaderType(...)
{
    GlobalListLink.LinkHead(GetTypeList());
}

现在再来看看IMPLEMENT_MATERIAL_SHADER_TYPE的展开

#define IMPLEMENT_MATERIAL_SHADER_TYPE(TemplatePrefix,ShaderClass,SourceFilename,FunctionName,Frequency) \
    IMPLEMENT_SHADER_TYPE( \
        TemplatePrefix, \
        ShaderClass, \
        SourceFilename, \
        FunctionName, \
        Frequency \
        );

#define IMPLEMENT_SHADER_TYPE(TemplatePrefix,ShaderClass,SourceFilename,FunctionName,Frequency) \
    TemplatePrefix \
    ShaderClass::ShaderMetaType ShaderClass::StaticType( \
        TEXT(#ShaderClass), \
        SourceFilename, \
        FunctionName, \
        Frequency, \
        1, \
        ShaderClass::ConstructSerializedInstance, \
        ShaderClass::ConstructCompiledInstance, \
        ShaderClass::ModifyCompilationEnvironment, \
        ShaderClass::ShouldCompilePermutation, \
        ShaderClass::ValidateCompiledResult \
        );

所以，

IMPLEMENT_MATERIAL_SHADER_TYPE(, FMobileXXXPS, TEXT("/Engine/Private/XXXPassShader.usf"), TEXT("MainPS"), SF_Pixel);

将会被展开为：

FMobileXXXPS::ShaderMetaType FMobileXXXPS::StaticType("XXXPassShader.usf", "MainPS", SF_Pixel);

其中，FMobileXXXPS::ShaderMetaType FMobileXXXPS::StaticType 在DECLARE_SHADER_TYPE 展开时，已经指定。

所以，这也是为什么使用定制自己的shader时要先

class FMobileXXXPS : public FMeshMaterialShader
{
    DECLARE_SHADER_TYPE(FMobileXXXPS, MeshMaterial);
public:
    ...
}

再

IMPLEMENT_MATERIAL_SHADER_TYPE(, FMobileXXXPS, TEXT("/Engine/Private/XXXPassShader.usf"), TEXT("MainPS"), SF_Pixel);

2.Shader Permutation

但以上只是讨论了几个宏的展开，没有涉及shader的变体，在ue4中，Shader Permutation又是如何运行的呢？
同样以TMobileBasePassPS 为例：

template< typename LightMapPolicyType, EOutputFormat OutputFormat, bool bEnableSkyLight, bool bRenderRuntimeProceduralTexture, int32 NumMovablePointLights>
class TMobileBasePassPS : public TMobileBasePassPSBaseType
{
    ...
}

可以看到TMobileBasePassPS是一个模板类，有四个模板参数：
LightMapPolicyType，各种PolicyType在LightMapRendering.h中定义，如light map和shadow等的渲染，
引擎中Indirect Light Cache、VLM、Distance Field Shadow都有对应的LightMapPolicy类
OutputFormat，PS的输出是gamma的还是linear
bEnableSkyLight，是否开启SkyLight
NumMovablePointLights，PS里要处理的动态点光源的数目这些参数在GetUniformMobileBasePassShaders中使用

template 
void GetUniformMobileBasePassShaders(
    const FMaterial& Material,
    FVertexFactoryType* VertexFactoryType,
    bool bEnableSkyLight,
    bool bRenderRuntimeProceduralTexture,
    TMobileBasePassVSPolicyParamType*& VertexShader,
    TMobileBasePassPSPolicyParamType*& PixelShader
    )
{
    if (IsMobileHDR())
    {
        VertexShader = (TMobileBasePassVSPolicyParamType*)Material.GetShader, HDR_LINEAR_64> >(VertexFactoryType);

        if (bEnableSkyLight)
        {
            if (bRenderRuntimeProceduralTexture)
            {
                PixelShader = (TMobileBasePassPSPolicyParamType*)Material.GetShader< TMobileBasePassPS, HDR_LINEAR_64, true, true, NumMovablePointLights> >(VertexFactoryType);
            }
            else
            {
                PixelShader = (TMobileBasePassPSPolicyParamType*)Material.GetShader< TMobileBasePassPS, HDR_LINEAR_64, true, false, NumMovablePointLights> >(VertexFactoryType);
            }
        }
        else
        {
            if (bRenderRuntimeProceduralTexture)
            {
                PixelShader = (TMobileBasePassPSPolicyParamType*)Material.GetShader< TMobileBasePassPS, HDR_LINEAR_64, false, true, NumMovablePointLights> >(VertexFactoryType);
            }
            else
            {
                PixelShader = (TMobileBasePassPSPolicyParamType*)Material.GetShader< TMobileBasePassPS, HDR_LINEAR_64, false, false, NumMovablePointLights> >(VertexFactoryType);
            }
        }
    }
    else
    {
        VertexShader = (TMobileBasePassVSPolicyParamType*)Material.GetShader, LDR_GAMMA_32> >(VertexFactoryType);

        if (bEnableSkyLight)
        {
            if (bRenderRuntimeProceduralTexture)
            {
                PixelShader = (TMobileBasePassPSPolicyParamType*)Material.GetShader< TMobileBasePassPS, LDR_GAMMA_32, true, true, NumMovablePointLights> >(VertexFactoryType);
            }
            else
            {
                PixelShader = (TMobileBasePassPSPolicyParamType*)Material.GetShader< TMobileBasePassPS, LDR_GAMMA_32, true, false, NumMovablePointLights> >(VertexFactoryType);
            }
        }
        else
        {
            if (bRenderRuntimeProceduralTexture)
            {
                PixelShader = (TMobileBasePassPSPolicyParamType*)Material.GetShader< TMobileBasePassPS, LDR_GAMMA_32, false, true, NumMovablePointLights> >(VertexFactoryType);
            }
            else
            {
                PixelShader = (TMobileBasePassPSPolicyParamType*)Material.GetShader< TMobileBasePassPS, LDR_GAMMA_32, false, false, NumMovablePointLights> >(VertexFactoryType);
            }
        }
    }
}

在运行时，根据HDR，SkyLight和LightMapPolicy等配置，调用Material的GetShader函数来得到不同的VS和PS类。每一个TMobileBasePassVS/PS类就对应一个Shader Permutation。

但是，这些Shader Permutation从何而来，大概多少个？

细看MobileBasePassRendering.cpp类

#define IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_VERTEX_SHADER_TYPE(LightMapPolicyType,LightMapPolicyName) \
    typedef TMobileBasePassVS< LightMapPolicyType, LDR_GAMMA_32 > TMobileBasePassVS##LightMapPolicyName##LDRGamma32; \
    typedef TMobileBasePassVS< LightMapPolicyType, HDR_LINEAR_64 > TMobileBasePassVS##LightMapPolicyName##HDRLinear64; \
    IMPLEMENT_MATERIAL_SHADER_TYPE(template<>, TMobileBasePassVS##LightMapPolicyName##LDRGamma32, TEXT("/Engine/Private/MobileBasePassVertexShader.usf"), TEXT("Main"), SF_Vertex); \
    IMPLEMENT_MATERIAL_SHADER_TYPE(template<>, TMobileBasePassVS##LightMapPolicyName##HDRLinear64, TEXT("/Engine/Private/MobileBasePassVertexShader.usf"), TEXT("Main"), SF_Vertex);

#define IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_PIXEL_SHADER_TYPE(LightMapPolicyType,LightMapPolicyName,NumMovablePointLights) \
    typedef TMobileBasePassPS< LightMapPolicyType, LDR_GAMMA_32, false, NumMovablePointLights > TMobileBasePassPS##LightMapPolicyName##NumMovablePointLights##LDRGamma32; \
    typedef TMobileBasePassPS< LightMapPolicyType, HDR_LINEAR_64, false, NumMovablePointLights > TMobileBasePassPS##LightMapPolicyName##NumMovablePointLights##HDRLinear64; \
    typedef TMobileBasePassPS< LightMapPolicyType, LDR_GAMMA_32, true, NumMovablePointLights > TMobileBasePassPS##LightMapPolicyName##NumMovablePointLights##LDRGamma32##Skylight; \
    typedef TMobileBasePassPS< LightMapPolicyType, HDR_LINEAR_64, true, NumMovablePointLights > TMobileBasePassPS##LightMapPolicyName##NumMovablePointLights##HDRLinear64##Skylight; \
    IMPLEMENT_MATERIAL_SHADER_TYPE(template<>, TMobileBasePassPS##LightMapPolicyName##NumMovablePointLights##LDRGamma32, TEXT("/Engine/Private/MobileBasePassPixelShader.usf"), TEXT("Main"), SF_Pixel); \
    IMPLEMENT_MATERIAL_SHADER_TYPE(template<>, TMobileBasePassPS##LightMapPolicyName##NumMovablePointLights##HDRLinear64, TEXT("/Engine/Private/MobileBasePassPixelShader.usf"), TEXT("Main"), SF_Pixel); \
    IMPLEMENT_MATERIAL_SHADER_TYPE(template<>, TMobileBasePassPS##LightMapPolicyName##NumMovablePointLights##LDRGamma32##Skylight, TEXT("/Engine/Private/MobileBasePassPixelShader.usf"), TEXT("Main"), SF_Pixel); \
    IMPLEMENT_MATERIAL_SHADER_TYPE(template<>, TMobileBasePassPS##LightMapPolicyName##NumMovablePointLights##HDRLinear64##Skylight, TEXT("/Engine/Private/MobileBasePassPixelShader.usf"), TEXT("Main"), SF_Pixel);

static_assert(MAX_BASEPASS_DYNAMIC_POINT_LIGHTS == 4, "If you change MAX_BASEPASS_DYNAMIC_POINT_LIGHTS, you need to add shader types below");

// Permutations for the number of point lights to support. INT32_MAX indicates the shader should use branching to support a variable number of point lights.
#define IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_SHADER_TYPE(LightMapPolicyType,LightMapPolicyName) \
    IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_VERTEX_SHADER_TYPE(LightMapPolicyType, LightMapPolicyName) \
    IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_PIXEL_SHADER_TYPE(LightMapPolicyType, LightMapPolicyName, 0) \
    IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_PIXEL_SHADER_TYPE(LightMapPolicyType, LightMapPolicyName, 1) \
    IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_PIXEL_SHADER_TYPE(LightMapPolicyType, LightMapPolicyName, 2) \
    IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_PIXEL_SHADER_TYPE(LightMapPolicyType, LightMapPolicyName, 3) \
    IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_PIXEL_SHADER_TYPE(LightMapPolicyType, LightMapPolicyName, 4) \
    IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_PIXEL_SHADER_TYPE(LightMapPolicyType, LightMapPolicyName, INT32_MAX)

// Implement shader types per lightmap policy
IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_SHADER_TYPE(TUniformLightMapPolicy, FNoLightMapPolicy);
IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_SHADER_TYPE(TUniformLightMapPolicy, TLightMapPolicyLQ);
IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_SHADER_TYPE(TUniformLightMapPolicy, FMobileDistanceFieldShadowsAndLQLightMapPolicy);
IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_SHADER_TYPE(TUniformLightMapPolicy, FMobileDistanceFieldShadowsLightMapAndCSMLightingPolicy);
IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_SHADER_TYPE(TUniformLightMapPolicy, FMobileDirectionalLightAndSHIndirectPolicy);
IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_SHADER_TYPE(TUniformLightMapPolicy, FMobileMovableDirectionalLightAndSHIndirectPolicy);
IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_SHADER_TYPE(TUniformLightMapPolicy, FMobileMovableDirectionalLightCSMAndSHIndirectPolicy);
IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_SHADER_TYPE(TUniformLightMapPolicy, FMobileDirectionalLightCSMAndSHIndirectPolicy);
IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_SHADER_TYPE(TUniformLightMapPolicy, FMobileMovableDirectionalLightLightingPolicy);
IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_SHADER_TYPE(TUniformLightMapPolicy, FMobileMovableDirectionalLightCSMLightingPolicy);
IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_SHADER_TYPE(TUniformLightMapPolicy, FMobileMovableDirectionalLightWithLightmapPolicy);
IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_SHADER_TYPE(TUniformLightMapPolicy, FMobileMovableDirectionalLightCSMWithLightmapPolicy);

在这里有详细的遂步展开，
IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_PIXEL_SHADER_TYPE，展开了与OutputFormat和bEnableSkyLight相关的各种情况， 4种

IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_SHADER_TYPE， 展开了各种动态光源的情况， （0，1，2，3，4，INT32_MAX）一共6种

IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_SHADER_TYPE， 展开了各种LightMapPolicy的情况， 12种
可以估算MobileBasePass材质的PS大概有：4*6*12=288个shader变种。

IMPLEMENT_MOBILE_SHADING_BASEPASS_LIGHTMAPPED_VERTEX_SHADER_TYPE， 展开了所有vs的情况， 2种
可以估算MobileBasePass材质大概有：2*12 =24 个shader变种。

所以一个MobileBasePass材质大概有：288+24 =312 个shader变种。

3.C++与HLSL的VertexFactory的绑定

前面说了宏对shader的绑定，而其中有一个宏还没说：IMPLEMENT_VERTEX_FACTORY_TYPE

静态物体ClipSpace坐标使用顶点Attribute属性输入的Position经MVP变换得到，蒙皮动画的物体则要在此之前做一次蒙皮计算。UE4把这个计算抽象成了VertexFactory文件，每一个VF文件都都实现VertexFactoryGetWorldPosition、VertexFactoryGetInterpolants等方法，PassVertexShader中再调用这些方法，这样，某些顶点计算便和PassShaders解耦。

因此，实现MeshPipeLine时，除了实现上述的PassShaders，还需要实现VertexFactory。Shader的变体和绑定正是上述的内容，而IMPLEMENT_VERTEX_FACTORY_TYPE正是将C++中的VertexFactory类与HLSL中的Vertex结构体绑定起来了，过程大同小异，不再展开。

现在知道了这些Shader Permutation从何而来了，那么该回过头来说说，当调用Material的GetShader函数后，是如何得到不同的VS和PS类了。

材质编译
 

往下阅读时，需要补充一下MeshPipleLine相关的知识：UE4 Mesh Piple Line & Auto Instacing

下面的内容依下面的流程进行：

VertexFactory.usf........|
passShader.usf...........|------> 离线编译GLSL源码---->Cook UMaterial------> 运行时编译及GLSL源码
材质连线.....................|

1.Cook

// 生成所需数据并缓存
void UMaterial::BeginCacheForCookedPlatformData( const ITargetPlatform *TargetPlatform )
{
    //DesiredShaderFormats--->当前平台和配置下所需要编译Shader格式
    for (int32 FormatIndex = 0; FormatIndex < DesiredShaderFormats.Num(); FormatIndex++)
    {
        CacheResourceShadersForCooking(LegacyShaderPlatform, *CachedMaterialResourcesForPlatform, TargetPlatform);
    }
}

void UMaterial::CacheResourceShadersForCooking(EShaderPlatform ShaderPlatform, TArray& OutCachedMaterialResources, const ITargetPlatform* TargetPlatform)
{
    //为当前ShaderFormat/QualityLevel生成FMaterialResource，
    for (int32 QualityLevelIndex = 0; QualityLevelIndex < EMaterialQualityLevel::Num; QualityLevelIndex++)
    {
        if (QualityLevelsUsed[QualityLevelIndex])
        {
            FMaterialResource* NewResource = AllocateResource();
            NewResource->SetMaterial(this, (EMaterialQualityLevel::Type)QualityLevelIndex, QualityLevelsUsed[QualityLevelIndex], (ERHIFeatureLevel::Type)TargetFeatureLevel);
            ResourcesToCache.Add(NewResource);
        }
    }
    //FMaterialResource存放到UMaterial::CachedMaterialResourcesForPlatform
    CacheShadersForResources(ShaderPlatform, ResourcesToCache, TargetPlatform);

    for (int32 ResourceIndex = 0; ResourceIndex < ResourcesToCache.Num(); ResourceIndex++)
    {
        OutCachedMaterialResources.Add(ResourcesToCache[ResourceIndex]);
    }
}

// 上述过程完成，usf文件和材质连线都通过CacheShadersForResources被转化成了FMaterialResource
// 这时，触发Serialize
void UMaterial::Serialize(FArchive& Ar)
{
    //序列化FMaterialResource，Cook到了UMaterial里面
}

2.FMaterialResource

以下几个类需要详细看看：

class ENGINE_VTABLE UMaterial : public UMaterialInterface
{
    // UMaterial里面存放着QualityLevelNum*FeatureLevelNum个FMaterialResource
    // 可以通过QualityLevel和FeatureLevel索引到FMaterialResource
    FMaterialResource* MaterialResources[EMaterialQualityLevel::Num][ERHIFeatureLevel::Num];
}

class ENGINE_VTABLE FMaterialResource : public FMaterial {...}


class ENGINE_VTABLE FMaterial
{
    // 此处，终于见到传说中的shadermap
    FMaterialShaderMap* RenderingThreadShaderMap;
    //给两个线程访问的FMaterialShaderMap
    TRefCountPtr GameThreadShaderMap;
    FMaterialShaderMap* RenderingThreadShaderMap;
}

//FMaterialShaderMap可以通过FVertexFactoryType索引到FMeshMaterialShaderMap
FShader* FMaterial::GetShader(FMeshMaterialShaderType* ShaderType, FVertexFactoryType* VertexFactoryType, int32 PermutationId, bool bFatalIfMissing) const
{
    const FMeshMaterialShaderMap* MeshShaderMap = RenderingThreadShaderMap->GetMeshShaderMap(VertexFactoryType);
    FShader* Shader = MeshShaderMap ? MeshShaderMap->GetShader(ShaderType, PermutationId) : nullptr;
    return Shader;
}

// FMaterialShaderMap存放着FMeshMaterialShaderMap数组，以VertexFactoryType::GetId()为Index
class FMaterialShaderMap : public TShaderMap, public FDeferredCleanupInterface
{
    const FMeshMaterialShaderMap* GetMeshShaderMap(FVertexFactoryType* VertexFactoryType) const
    {
        const FMeshMaterialShaderMap* MeshShaderMap = OrderedMeshShaderMaps[VertexFactoryType->GetId()];
        return MeshShaderMap;
    }

    TArray OrderedMeshShaderMaps;
}

// FMeshMaterialShaderMap继承自TShaderMap，存放着FShader Map，按ShaderType索引
class FMeshMaterialShaderMap : public TShaderMap {...}
class TShaderMap
{
    TMap > Shaders;
}

//FMeshMaterialShaderMap可以通过FShaderType索引FShader。
FShader* TShaderMap::GetShader(FShaderType* ShaderType, int32 PermutationId = 0) const
{
    check(bHasBeenRegistered);
    const TRefCountPtr* ShaderRef = Shaders.Find(FShaderPrimaryKey(ShaderType, PermutationId));
    return ShaderRef ? (*ShaderRef)->GetShader() : nullptr;
}

从上面的代码可以看到这样的内存结构：UMaterial里面存放着QualityLevelNum*FeatureLevelNum个FMaterialResource，可以通过QualityLevel和FeatureLevel索引到FMaterialResource

FMaterialResource里有FMaterialShaderMap

FMaterialShaderMap可以通过FVertexFactoryType::GetId()索引FMeshMaterialShaderMap

FMeshMaterialShaderMap可以通过FShaderType来索引FShader

所以，FMaterialResource里面存放的是FShader的集合，如下：
 

UMaterial -----------------> [QualityLevelNum*FeatureLevelNum]FMaterialResource
FMaterialResource---------->FMaterialShaderMap
FMaterialShaderMap--------->[FVertexFactoryType]FMeshMaterialShaderMap
FMeshMaterialShaderMap----->[FShaderType]FShader

3.FShader

class RENDERCORE_API FShader : public FDeferredCleanupInterface
{
    TRefCountPtr Resource;
    FVertexFactoryType* VFType;
    FShaderType* Type;
}

class FShaderResource : public FRenderResource, public FDeferredCleanupInterface
{
    /** Compiled bytecode. */
    TArray Code;
}

FShader::Resource::Code---->最终渲染需要用到的压缩过的Shader源码

总结：

UMaterial--->上述的Permutation机制--->PassShaders,VertexFactory有Permutation,各种宏组合生成代码--->按上面讲的结构存储

有了转换后的数据，便开始讲讲编译了

Shader编译

1.收集FShaderCompileJob

根据shader格式、usf路径、注入宏等等收集FShaderCompileJob

bool FMaterial::BeginCompileShaderMap(...)
{
    NewShaderMap->Compile(this, ShaderMapId, MaterialEnvironment, NewCompilationOutput, Platform, bSynchronousCompile);
    OutShaderMap = NewShaderMap->CompiledSuccessfully() ? NewShaderMap : nullptr;
}

void FMaterialShaderMap::Compile()
{
    TArray NewJobs;

    // 遍历所有的FVertexFactoryType
    for(TLinkedList::TIterator VertexFactoryTypeIt(FVertexFactoryType::GetTypeList());VertexFactoryTypeIt;VertexFactoryTypeIt.Next())
    {
        const uint32 MeshShaders = MeshShaderMap->BeginCompile(
            Material,
            MaterialEnvironment,
            NewJobs,
            ...
            );
    }

    // 收集完毕，真正开始编译
    GShaderCompilingManager->AddJobs(NewJobs, bSynchronousCompile || !Material->IsPersistent(), bRecreateComponentRenderStateOnCompletion);
    if (bSynchronousCompile)
    {
        GShaderCompilingManager->FinishCompilation();
    }
}

uint32 FMeshMaterialShaderMap::BeginCompile(...)
{
    // 遍历所有的FShaderType
    for (TLinkedList::TIterator ShaderTypeIt(FShaderType::GetTypeList());ShaderTypeIt;ShaderTypeIt.Next())
    {
        FMeshMaterialShaderType* ShaderType = ShaderTypeIt->GetMeshMaterialShaderType();
        ...
        // 最终会调到PassShader类的ShouldCompilePermutation
        if (ShouldCacheMeshShader(ShaderType, InPlatform, Material, VertexFactoryType, PermutationId))
        {
            auto* Job = ShaderType->BeginCompileShader(
                Material,
                MaterialEnvironment,
                VertexFactoryType,
                NewJobs
                );
        }
    }
}

FShaderCompileJob* FMeshMaterialShaderType::BeginCompileShader(...)
{
    FShaderCompileJob* NewJob = new FShaderCompileJob(ShaderMapId, VertexFactoryType, this, PermutationId);
    NewJob->Input.SharedEnvironment = MaterialEnvironment;

    // 最终会调到PassShader类的ModifyCompilationEnvironment
    (*ModifyCompilationEnvironmentRef)(FMaterialShaderPermutationParameters(Platform, Material, PermutationId), Environment);

    ::GlobalBeginCompileShader(
        NewJob,
        NewJobs,
        ...
        );
    return NewJob;
}

void GlobalBeginCompileShader(...)
{
    FShaderCompilerInput& Input = NewJob->Input;
    Input.Target = Target;
    Input.ShaderFormat = LegacyShaderPlatformToShaderFormat(EShaderPlatform(Target.Platform));
    Input.VirtualSourceFilePath = SourceFilename;
    ...
    Input.Environment.SetDefine(TEXT("PROJECT_SUPPORT_VIRTUAL_TEXTURE"), bVirtualTextureEnabled ? 1 : 0);
    Input.Environment.SetDefine(TEXT("PROJECT_MOBILE_USE_LEGACY_SHADING"), CVar ? (CVar->GetInt() != 0) : 0);
    ...
    NewJobs.Add(NewJob);
}

遍历的FVertexFactoryType，FShaderType，GlobalBeginShader，收集FShaderCompileJob， 这里将会调用PassShader类的ModifyCompilationEnvironment，ShouldCompilePermutation，
这也是为何每个PassShader都要实现此类接口才能编译通过。

真正开始编译是发生在这里

void FMaterialShaderMap::Compile()
{
    // 收集完毕，真正开始编译
    GShaderCompilingManager->AddJobs(NewJobs, bSynchronousCompile || !Material->IsPersistent(), bRecreateComponentRenderStateOnCompletion);
    if (bSynchronousCompile)
    {
        GShaderCompilingManager->FinishCompilation();
    }
}

编译过程---->此处省去几万字(其实是项目太忙，还没啃完)--->FShader::Resource::Code塞进了FShaderCompilerOutput::ShaderCode压缩后的内容----->把FShader塞进FMeshMaterialMap------>编辑器期的编译完成

2.RenderPass Get Shader

FDefaultMaterialInstance：

FDefaultMaterialInstance继承自FMaterialRenderProxy，是UMaterial在渲染线程的代理，为Renderer提供材质相关的一切信息

UMaterial在运行时被加载时--->把FMaterialResource序列化到LoadedMaterialResources成员中--->将LoadedMaterialResources根据QualityLevel、FeatureLevel分类存放到MaterialResources成员中--->生成FDefaultMaterialInstance（此处同样省去几万字）

往下阅读时，需要补充一下MeshPipleLine相关的知识：UE4 Mesh Piple Line & Auto Instacing

不看也行，只要记住以下就可：
FMeshBatch是一个MeshSection在RenderThread的完整数据集合，包含了顶点、索引、材质、VertexFactory等信息，材质的载体便是上面提到的FMaterialRenderProxy。所有的Pass都从FMeshBatch获取到需要的信息来构建FMeshDrawCommand。

void FPrimitiveSceneInfo::AddToScene(FRHICommandListImmediate& RHICmdList, bool bUpdateStaticDrawLists, bool bAddToStaticDrawLists)
{
    MeshSceneProxy->DrawStaticElements(&BatchingSPDI);
    for (int32 MeshIndex = 0; MeshIndex < StaticMeshes.Num(); MeshIndex++)
    {
        for (int32 PassIndex = 0; PassIndex < EMeshPass::Num; PassIndex++)
        {
            EMeshPass::Type PassType = (EMeshPass::Type)PassIndex;
            PassProcessorCreateFunction CreateFunction = FPassProcessorManager::GetCreateFunction(ShadingPath, PassType);
            FMeshPassProcessor* PassMeshProcessor = CreateFunction(Scene, nullptr, &CachedPassMeshDrawListContext);
            PassMeshProcessor->AddMeshBatch(Mesh, BatchElementMask, MeshSceneProxy);
        }
    }
}

void FStaticMeshSceneProxy::DrawStaticElements(FStaticPrimitiveDrawInterface* PDI)
{
    if (GetMeshElement(LODIndex, BatchIndex, SectionIndex, PrimitiveDPG, bIsMeshElementSelected, true, BaseMeshBatch))
    {
        PDI->DrawMesh(BaseMeshBatch, FLT_MAX);
    }
}

bool FStaticMeshSceneProxy::GetMeshElement(...)
{
    // 取到FMaterialRenderProxy
    FMaterialRenderProxy* MaterialRenderProxy = MaterialInterface->GetRenderProxy();
}

void FMobileBasePassMeshProcessor::AddMeshBatch(const FMeshBatch& RESTRICT MeshBatch, const FPrimitiveSceneProxy* RESTRICT PrimitiveSceneProxy ...)
{

    TMeshProcessorShaders<
        TMobileBasePassVSPolicyParamType,
        FBaseHS,
        FBaseDS,
        TMobileBasePassPSPolicyParamType> BasePassShaders;

    // 获取所有Stage的FShader
    MobileBasePass::GetShaders(
        LightMapPolicyType,
        NumMovablePointLights,
        MaterialResource,
        MeshBatch.VertexFactory->GetType(),
        bEnableSkyLight,
        BasePassShaders.VertexShader,
        BasePassShaders.PixelShader);

    // 得到FShader后，构建FMeshDrawCommand
    BuildMeshDrawCommands(
        MeshBatch,
        BatchElementMask,
        PrimitiveSceneProxy,
        MaterialRenderProxy,
        MaterialResource,
        DrawRenderState,
        BasePassShaders,
        MeshFillMode,
        MeshCullMode,
        SortKey,
        EMeshPassFeatures::Default,
        ShaderElementData);
}


// 获取所有Stage的FShader
MobileBasePass::GetShaders(
    LightMapPolicyType,
    NumMovablePointLights,
    MaterialResource,
    MeshBatch.VertexFactory->GetType(),
    bEnableSkyLight,
    BasePassShaders.VertexShader,
    BasePassShaders.PixelShader);

template 
void GetUniformMobileBasePassShaders(
	const FMaterial& Material, 
	FVertexFactoryType* VertexFactoryType, 
	bool bEnableSkyLight,
	bool bRenderRuntimeProceduralTexture,
	TMobileBasePassVSPolicyParamType*& VertexShader,
	TMobileBasePassPSPolicyParamType*& PixelShader
	)
{
	if (IsMobileHDR())
	{
		VertexShader = (TMobileBasePassVSPolicyParamType*)Material.GetShader, HDR_LINEAR_64> >(VertexFactoryType);

		if (bEnableSkyLight)
		{
			if (bRenderRuntimeProceduralTexture)
			{
				PixelShader = (TMobileBasePassPSPolicyParamType*)Material.GetShader< TMobileBasePassPS, HDR_LINEAR_64, true, true, NumMovablePointLights> >(VertexFactoryType);
			}
			else
			{
				PixelShader = (TMobileBasePassPSPolicyParamType*)Material.GetShader< TMobileBasePassPS, HDR_LINEAR_64, true, false, NumMovablePointLights> >(VertexFactoryType);
			}
		}
		else
		{
			if (bRenderRuntimeProceduralTexture)
			{
				PixelShader = (TMobileBasePassPSPolicyParamType*)Material.GetShader< TMobileBasePassPS, HDR_LINEAR_64, false, true, NumMovablePointLights> >(VertexFactoryType);
			}
			else
			{
				PixelShader = (TMobileBasePassPSPolicyParamType*)Material.GetShader< TMobileBasePassPS, HDR_LINEAR_64, false, false, NumMovablePointLights> >(VertexFactoryType);
			}
		}	
	}
	else
	{
		VertexShader = (TMobileBasePassVSPolicyParamType*)Material.GetShader, LDR_GAMMA_32> >(VertexFactoryType);

		if (bEnableSkyLight)
		{
			if (bRenderRuntimeProceduralTexture)
			{
				PixelShader = (TMobileBasePassPSPolicyParamType*)Material.GetShader< TMobileBasePassPS, LDR_GAMMA_32, true, true, NumMovablePointLights> >(VertexFactoryType);
			}
			else
			{
				PixelShader = (TMobileBasePassPSPolicyParamType*)Material.GetShader< TMobileBasePassPS, LDR_GAMMA_32, true, false, NumMovablePointLights> >(VertexFactoryType);
			}
		}
		else
		{
			if (bRenderRuntimeProceduralTexture)
			{
				PixelShader = (TMobileBasePassPSPolicyParamType*)Material.GetShader< TMobileBasePassPS, LDR_GAMMA_32, false, true, NumMovablePointLights> >(VertexFactoryType);
			}
			else
			{
				PixelShader = (TMobileBasePassPSPolicyParamType*)Material.GetShader< TMobileBasePassPS, LDR_GAMMA_32, false, false, NumMovablePointLights> >(VertexFactoryType);
			}
		}			
	}
}

也即AddMeshBatch时传入相应参数，取得对应的shader,这些上述的Shader Permutation机制已讲。

总结

编译期所有shader类型展开与绑定好，运行时传参获取。

再后面便是运行时的编译了，再熟悉不过的opengl的glCreateShader，glShaderSource等接口，shader编译，shader链接,生成shaderobj,并设置到FRHIShader使用。（此处用代码来省去几万字）

template
void FMeshPassProcessor::BuildMeshDrawCommands(
    const FMeshBatch& RESTRICT MeshBatch,
    uint64 BatchElementMask,
    const FPrimitiveSceneProxy* RESTRICT PrimitiveSceneProxy,
    const FMaterialRenderProxy& RESTRICT MaterialRenderProxy,
    const FMaterial& RESTRICT MaterialResource,
    const FMeshPassProcessorRenderState& RESTRICT DrawRenderState,
    PassShadersType PassShaders,
    ERasterizerFillMode MeshFillMode,
    ERasterizerCullMode MeshCullMode,
    FMeshDrawCommandSortKey SortKey,
    EMeshPassFeatures MeshPassFeatures,
    const ShaderElementDataType& ShaderElementData)
{
    const FVertexFactory* RESTRICT VertexFactory = MeshBatch.VertexFactory;
    const FPrimitiveSceneInfo* RESTRICT PrimitiveSceneInfo = PrimitiveSceneProxy ? PrimitiveSceneProxy->GetPrimitiveSceneInfo() : nullptr;

    FMeshDrawCommand SharedMeshDrawCommand;
    // ...
    SharedMeshDrawCommand.SetShaders(VertexDeclaration, PassShaders.GetUntypedShaders(), PipelineState);

    PipelineState.RasterizerState = GetStaticRasterizerState(MeshFillMode, MeshCullMode);


    check(VertexFactory && VertexFactory->IsInitialized());
    VertexFactory->GetStreams(FeatureLevel, InputStreamType, SharedMeshDrawCommand.VertexStreams);

    SharedMeshDrawCommand.PrimitiveIdStreamIndex = VertexFactory->GetPrimitiveIdStreamIndex(InputStreamType);

    }
}


void FMeshDrawCommand::SetShaders(FRHIVertexDeclaration* VertexDeclaration, const FMeshProcessorShaders& Shaders, FGraphicsMinimalPipelineStateInitializer& PipelineState)
{
    PipelineState.BoundShaderState = FBoundShaderStateInput(
        VertexDeclaration
        , GETSAFERHISHADER_VERTEX(Shaders.VertexShader)
#if PLATFORM_SUPPORTS_TESSELLATION_SHADERS
        , GETSAFERHISHADER_HULL(Shaders.HullShader)
        , GETSAFERHISHADER_DOMAIN(Shaders.DomainShader)
#endif
        , GETSAFERHISHADER_PIXEL(Shaders.PixelShader)
#if PLATFORM_SUPPORTS_GEOMETRY_SHADERS
        , GETSAFERHISHADER_GEOMETRY(Shaders.GeometryShader)
#endif
    );

    ShaderBindings.Initialize(Shaders);
}


// the following helper macros allow to safely convert shader types without much code clutter
#define GETSAFERHISHADER_PIXEL(Shader) ((Shader) ? (Shader)->GetPixelShader() : nullptr)
#define GETSAFERHISHADER_VERTEX(Shader) ((Shader) ? (Shader)->GetVertexShader() : nullptr)

/** @return the shader's vertex shader */
FORCEINLINE FRHIVertexShader* GetVertexShader()
{
    checkSlow(Target.Frequency == SF_Vertex);
    if (!IsInitialized())
    {
        InitializeShaderRHI();
    }
    return (FRHIVertexShader*)Shader.GetReference();
}
/** @return the shader's pixel shader */
FORCEINLINE FRHIPixelShader* GetPixelShader()
{
    checkSlow(Target.Frequency == SF_Pixel);
    if (!IsInitialized())
    {
        InitializeShaderRHI();
    }
    return (FRHIPixelShader*)Shader.GetReference();
}

// FRHIVertexShader, FRHIPixelShader继承自FRHIShader，持有Opengl的Shader Object的句柄


void FShaderResource::InitRHI()
{
    // 取得Cook压缩后的glsl源码
    TArray UncompressedCode;
    if (!bCodeInSharedLocation)
    {
        UncompressCode(UncompressedCode);
    }


    if(Target.Frequency == SF_Vertex)
    {
        Shader = FShaderCodeLibrary::CreateVertexShader((EShaderPlatform)Target.Platform, OutputHash, UncompressedCode);
        UE_CLOG((bCodeInSharedLocation && !IsValidRef(Shader)), LogShaders, Fatal, TEXT("FShaderResource::SerializeShaderCode can't find shader code for: [%s]"), *LegacyShaderPlatformToShaderFormat((EShaderPlatform)Target.Platform).ToString());
    }
    else if(Target.Frequency == SF_Pixel)
    {
        Shader = FShaderCodeLibrary::CreatePixelShader((EShaderPlatform)Target.Platform, OutputHash, UncompressedCode);
        UE_CLOG((bCodeInSharedLocation && !IsValidRef(Shader)), LogShaders, Fatal, TEXT("FShaderResource::SerializeShaderCode can't find shader code for: [%s]"), *LegacyShaderPlatformToShaderFormat((EShaderPlatform)Target.Platform).ToString());
    }
    // ...
}

FPixelShaderRHIRef FShaderCodeLibrary::CreatePixelShader(EShaderPlatform Platform, FSHAHash Hash, TArray const& Code)
{
    Shader = RHICreatePixelShader(Code);
    return Shader;
}

FPixelShaderRHIRef FOpenGLDynamicRHI::RHICreatePixelShader(const TArray& Code)
{
    return CreateProxyShader(Code);
}

template
RHIType* CreateProxyShader(const TArray& Code)
{
    return CompileOpenGLShader(Code, FSHAHash(), OwnerRHI);
}

template 
ShaderType* CompileOpenGLShader(const TArray& InShaderCode, const FSHAHash& LibraryHash, FRHIShader* RHIShader = nullptr)
{

    GLuint Resource = glCreateShader(Type);
    CompileCurrentShader(Resource, GlslCode);

    ShaderType* Shader = new ShaderType();
    Shader->Resource = Resource;

    return Shader;
}

GLint CompileCurrentShader(const GLuint Resource, const FAnsiCharArray& GlslCode)
{
    glShaderSource(Resource, 1, (const GLchar**)&GlslCodeString, &GlslCodeLength);
}