Studying note of GCC-3.4.6 source (178)

5.13.5.3.2.2.3.          Inline normal function

After handling function demanding inlining (by using “always_inline”), inlining rest functions are up to the compiler. As long as we don’t prohbit function by inlining option “-fno-inline” explicitly, flag_really_no_inline at line 1326 is 0.

 

cgraph_decide_inline (continue)

 

1320 #ifdef ENABLE_CHECKING

1321   for (node = cgraph_nodes ; node; node = node->next)

1322     if (node->aux || node->output)

1323       abort ();

1324 #endif

1325

1326   if (!flag_really_no_inline )

1327   {

1328     cgraph_decide_inlining_of_small_functions (inlined, inlined_callees);

1329 #ifdef ENABLE_CHECKING

1330     for (node = cgraph_nodes ; node; node = node->next)

1331       if (node->aux || node->output)

1332         abort ();

1333 #endif

1334

1335     if (cgraph_dump_file )

1336       fprintf (cgraph_dump_file , "/nDeciding on functions called once:/n");

 

Here, parameter inlined and inlined_callees are used as storage, their contents set previously are irrelevant, and will be overwirtten in late. In below analysis, it uses Fibonacci heaps to sort function by instruction number after expansion. The detail of Fibonacci heaps is not covered here.

 

1100       static void

1101       cgraph_decide_inlining_of_small_functions (struct cgraph_node **inlined,      in cgraphunit.c

1102                                            struct cgraph_node **inlined_callees)

1103       {

1104         int i;

1105         struct cgraph_node *node;

1106         fibheap_t heap = fibheap_new ();

1107         struct fibnode **heap_node =

1108           xcalloc (cgraph_max_uid , sizeof (struct fibnode *));

1109         int ninlined, ninlined_callees;

1110   int max_insns = ((HOST_WIDEST_INT) initial_insns

1111               * (100 + PARAM_VALUE (PARAM_INLINE_UNIT_GROWTH)) / 100);

1112

1113   /* Put all inline candidates into the heap.  */

1114

1115   for (node = cgraph_nodes ; node; node = node->next)

1116   {

1117     if (!node->local.inlinable || !node->callers

1118         || node->local.disregard_inline_limits)

1119       continue ;

1120      

1121           if (!cgraph_default_inline_p (node))

1122           {

1123              cgraph_set_inline_failed (node,

1124                                    N_("--param max-inline-insns-single limit reached"));

1125              continue ;

1126           }

1127           heap_node[node->uid] =

1128                 fibheap_insert (heap, cgraph_estimate_growth (node), node);

1129         }

 

As it is the compiler to make the decision, then if the function should be inlined, besides the inlinability, also depends on its size after expanding. For function declaring “inline”, this limitation is MAX_INLINE_INSNS_SINGLE (500 by default), while for other functions, the limitation is  MAX_INLINE_INSNS_AUTO (100 by default). Note that, the unit is instruction, not line.

 

1068 static bool

1069 cgraph_default_inline_p (struct cgraph_node *n)                                   in cgraphunit.c

1070 {

1071   if (!DECL_INLINE (n->decl) || !DECL_SAVED_TREE (n->decl))

1072     return false;

1073   if (DECL_DECLARED_INLINE_P (n->decl))

1074     return n->global.insns < MAX_INLINE_INSNS_SINGLE;

1075   else

1076     return n->global.insns < MAX_INLINE_INSNS_AUTO;

1077 }

 

If function fails in inlining, it needs to tell the caller about the failure reason, unless the caller has been inline forcely (at that time, its inline_failed is NULL).

 

1081 static void

1082 cgraph_set_inline_failed (struct cgraph_node *node, const char *reason)        in cgraphunit.c

1083 {

1084   struct cgraph_edge *e;

1085

1086   if (cgraph_dump_file )

1087     fprintf (cgraph_dump_file , "Inlining failed: %s/n", reason);

1088   for (e = node->callers; e; e = e->next_caller)

1089     if (e->inline_failed)

1090       e->inline_failed = reason;

1091 }

 

Once the size of the function is within the limitation, add it into Fibonacci heaps, but what used as the key value is the increment of instruction number dues to its expansion.

 

918  static int

919  cgraph_estimate_growth (struct cgraph_node *node)                             in cgraphunit.c

920  {

921    int growth = 0;

922    int calls_saved = 0;

923    int clones_added = 0;

924    struct cgraph_edge *e;

925 

926    for (e = node->callers; e; e = e->next_caller)

927      if (e->inline_failed)

928      {

929        growth += ((cgraph_estimate_size_after_inlining (1, e->caller, node)

930                   -

931                   e->caller->global.insns) *e->caller->global.cloned_times);

932        calls_saved += e->caller->global.cloned_times;

933        clones_added += e->caller->global.cloned_times;

934      }

935 

936    /* ??? Wrong for self recursive functions or cases where we decide to not

937       inline for different reasons, but it is not big deal as in that case

938      we will keep the body around, but we will also avoid some inlining.  */

939    if (!node->needed && !node->origin && !DECL_EXTERNAL (node->decl))

940      growth -= node->global.insns, clones_added--;

941 

942    if (!calls_saved)

943      calls_saved = 1;

944 

945    return growth;

946  }

 

However as now the analysises of its callers, callees haven’t been done, it is a rough estimation (not include the size of functions called directly or indirectly). With the advancing of below function analysis, the estimation approaches correct result. Because in below it will frequently overwrite the key value of nodes, out of efficiency consideration, use Fibonacci heaps.

At line 1133, fibheap_extract_min removes node of minus key value from Fibonacci heaps. At beginning, this value is just the instruction number of the function multiplies with the times it is invoked. Obviously, beginning at function with smallest key value is a good start, it is most possible the function at bottom of the call stack.

 

cgraph_decide_inlining_of_small_functions (continue)

 

1131         if (cgraph_dump_file )

1132           fprintf (cgraph_dump_file , "/nDeciding on smaller functions:/n");

1133         while (overall_insns <= max_insns && (node = fibheap_extract_min (heap)))

1134         {

1135           struct cgraph_edge *e;

1136           int old_insns = overall_insns ;

1137      

1138           heap_node[node->uid] = NULL;

1139           if (cgraph_dump_file )

1140             fprintf (cgraph_dump_file ,

1141                    "/nConsidering %s with %i insns/n"

1142                    " Estimated growth is %+i insns./n",

1143                    cgraph_node_name (node), node->global.insns,

1144                    cgraph_estimate_growth (node));

1145           if (!cgraph_default_inline_p (node))

1146           {

1147              cgraph_set_inline_failed (node,

1148                 N_("--param max-inline-insns-single limit reached after inlining into the callee"));

1149              continue ;

1150           }

1151           ninlined_callees = cgraph_inlined_callees (node, inlined_callees);

1152           for (e = node->callers; e; e = e->next_caller)

1153             if (e->inline_failed)

1154                {

1155                  /* Marking recursive function inlinine has sane semantic and

1156                   thus we should not warn on it.  */

1157                  if (e->caller == node)

1158                {

1159                  e->inline_failed = "";

1160                 continue ;

1161                 }

1162                ninlined = cgraph_inlined_into (e->caller, inlined);

1163                  if (e->callee->output)

1164                    e->inline_failed = "";

1165                  if (e->callee->output

1166                  || !cgraph_check_inline_limits (e->caller, node, inlined,

1167                                            ninlined, &e->inline_failed))

1168                 {

1169                 for (i = 0; i < ninlined; i++)

1170                    inlined[i]->output = 0, inlined[i]->aux = 0;

1171                 if (cgraph_dump_file )

1172                    fprintf (cgraph_dump_file , " Not inlining into %s./n",

1173                          cgraph_node_name (e->caller));

1174                 continue ;

1175                 }

1176                  cgraph_mark_inline (e->caller, node, inlined, ninlined,

1177                                inlined_callees, ninlined_callees);

1178                  if (heap_node[e->caller->uid])

1179                    fibheap_replace_key (heap, heap_node[e->caller->uid],

1180                                    cgraph_estimate_growth (e->caller));

1181      

1182                /* Size of the functions we updated into has changed, so update

1183                     the keys.  */

1184                   for (i = 0; i < ninlined; i++)

1185                 {

1186                 inlined[i]->output = 0, inlined[i]->aux = 0;

1187                 if (heap_node[inlined[i]->uid])

1188                    fibheap_replace_key (heap, heap_node[inlined[i]->uid],

1189                                      cgraph_estimate_growth (inlined[i]));

1190                  }

1191                  if (cgraph_dump_file )

1192                    fprintf (cgraph_dump_file ,

1193                        " Inlined into %s which now has %i insns./n",

1194                          cgraph_node_name (e->caller),

1195                          e->caller->global.insns);

1196                }

1197

1198           /* Similarly all functions called by the function we just inlined

1199              are now called more times; update keys.  */

1200      

1201     for (e = node->callees; e; e = e->next_callee)

1202       if (e->inline_failed && heap_node[e->callee->uid])

1203          fibheap_replace_key (heap, heap_node[e->callee->uid],

1204                            cgraph_estimate_growth (e->callee));

1205

1206     for (i = 0; i < ninlined_callees; i++)

1207     {

1208        struct cgraph_edge *e;

1209

1210         for (e = inlined_callees[i]->callees; e; e = e->next_callee)

1211                  if (e->inline_failed && heap_node[e->callee->uid])

1212            fibheap_replace_key (heap, heap_node[e->callee->uid],

1213                              cgraph_estimate_growth (e->callee));

1214

1215        inlined_callees[i]->output = 0;

1216        inlined_callees[i]->aux = 0;

1217     }

1218     if (cgraph_dump_file )

1219       fprintf (cgraph_dump_file ,

1220              " Inlined %i times for a net change of %+i insns./n",

1221              node->global.cloned_times, overall_insns - old_insns);

1222   }

1223   while ((node = fibheap_extract_min (heap)) != NULL)

1224     if (!node->local.disregard_inline_limits)

1225       cgraph_set_inline_failed (node, N_("--param inline-unit-growth limit reached"));

1226   fibheap_delete (heap);

1227   free (heap_node);

1228 }

 

Except giving priority to function of smaller expanding size, the compiler inlines function in the same way, once function expansion arrives threshold, no inlining is allowed anymore. The criterion, one way is the total increment of the instruction number, it is controlled by max_insns at line 1133 (by default, compiler allows increasing instruction number by 50%); on the other way is shown by cgraph_check_inline_limits .

 

1018 static bool

1019 cgraph_check_inline_limits (struct cgraph_node *to, struct cgraph_node *what,

1020                         struct cgraph_node **inlined, int ninlined,

1021                          const char **reason)

1022 {

1023   int i;

1024   int times = 0;

1025   struct cgraph_edge *e;

1026   int newsize;

1027   int limit;

1028

1029   for (e = to->callees; e; e = e->next_callee)

1030     if (e->callee == what)

1031       times++;

1032

1033   /* When inlining large function body called once into small function,

1034     take the inlined function as base for limiting the growth.  */

1035   if (to->local.self_insns > what->local.self_insns)

1036     limit = to->local.self_insns;

1037   else

1038     limit = what->local.self_insns;

1039

1040   limit += limit * PARAM_VALUE (PARAM_LARGE_FUNCTION_GROWTH) / 100;

1041

1042   newsize = cgraph_estimate_size_after_inlining (times, to, what);

1043   if (newsize > PARAM_VALUE (PARAM_LARGE_FUNCTION_INSNS)

1044       && newsize > limit)

1045   {

1046     *reason = N_("--param large-function-growth limit reached");

1047     return false;

1048   }

1049   for (i = 0; i < ninlined; i++)

1050   {

1051     newsize =

1052        cgraph_estimate_size_after_inlining (INLINED_TIMES (inlined[i]) *

1053                                       times, inlined[i], what);

1054     if (newsize > PARAM_VALUE (PARAM_LARGE_FUNCTION_INSNS)

1055          && newsize >

1056            inlined[i]->local.self_insns *

1057            (100 + PARAM_VALUE (PARAM_LARGE_FUNCTION_GROWTH)) / 100)

1058     {

1059        *reason = N_("--param large-function-growth limit reached while inlining the caller");

1060        return false;

1061     }

1062   }

1063   return true;

1064 }

 

PARAM_LARGE_FUNCTION_GROWTH is used to control instruction number increment in inlining large function, in percent (100% by default), obviously if a large function is inlined for more than 1 times, it will exceed this constrain. And PARAM_LARGE_FUNCTION_INSNS is a value, when 87 estimated instruction number is larger than it, the function is considered as larger function.

output at line 1163, in current situation, it is set by cgraph_inlined_callees at line 1151 as output is 0 oringinally, and in loop at line 1152, after every treatment it would reset ouput field of relative node (line 1170 and 1186), so if now find that output of the funciton has been set, obvious there is other functions inlines this function, then current function can’t be inlined by other callers (but it would still inline this function) to prevent the times of expansion of this functin from increasing exponently.

As long as the function passes check at line 1165 and 1166, it is regarded as can be inlined, and by cgraph_mark_inline to update the parameter. As result, the cost to expand it changes also, so needs unpdate the node in Heaps too (if it is still there).

In cgraph_decide_inlining_of_small_functions at last, if the increment of instruction numbe exceeds the restrain, then left functions aren’t allowed inlining. As heaps sorts functions by expanded instruction number, can image those left are relative large ones.

 

cgraph_decide_inline (continue)

 

1338     /* And finally decide what functions are called once.  */

1339

1340     for (i = nnodes - 1; i >= 0; i--)

1341     {

1342        node = order[i];

1343

1344        if (node->callers && !node->callers->next_caller && !node->needed

1345            && node->local.inlinable && node->callers->inline_failed

1346            && !DECL_EXTERNAL (node->decl) && !DECL_COMDAT (node->decl))

1347        {

1348          bool ok = true;

1349          struct cgraph_node *node1;

1350

1351          /* Verify that we won't duplicate the caller.  */

1352          for (node1 = node->callers->caller;

1353              node1->callers && !node1->callers->inline_failed

1354              && ok; node1 = node1->callers->caller)

1355           if (node1->callers->next_caller || node1->needed)

1356              ok = false;

1357          if (ok)

1358         {

1359            const char *dummy_reason;

1360            if (cgraph_dump_file )

1361              fprintf (cgraph_dump_file ,

1362                     "/nConsidering %s %i insns./n"

1363                     " Called once from %s %i insns./n",

1364                      cgraph_node_name (node), node->global.insns,

1365                      cgraph_node_name (node->callers->caller),

1366                      node->callers->caller->global.insns);

1367            ninlined = cgraph_inlined_into (node->callers->caller,

1368                                        inlined);

1369            old_insns = overall_insns ;

1370

1371            /* Inlining functions once would never cause inlining warnings.  */

1372            if (cgraph_check_inline_limits

1373                (node->callers->caller, node, inlined, ninlined,

1374                 &dummy_reason))

1375            {

1376              ninlined_callees =

1377                    cgraph_inlined_callees (node, inlined_callees);

1378              cgraph_mark_inline (node->callers->caller, node, inlined,

1379                                ninlined, inlined_callees,

1380                                ninlined_callees);

1381              for (y = 0; y < ninlined_callees; y++)

1382               inlined_callees[y]->output = 0, inlined_callees[y]->aux = 0;

1383              if (cgraph_dump_file )

1384               fprintf (cgraph_dump_file ,

1385                      " Inlined into %s which now has %i insns"

1386                      " for a net change of %+i insns./n",

1387                      cgraph_node_name (node->callers->caller),

1388                      node->callers->caller->global.insns,

1389                      overall_insns - old_insns);

1390            }

1391            else

1392            {

1393              if (cgraph_dump_file )

1394               fprintf (cgraph_dump_file ,

1395                      " Inline limit reached, not inlined./n");

1396            }

1397            for (y = 0; y < ninlined; y++)

1398              inlined[y]->output = 0, inlined[y]->aux = 0;

1399         }

1400        }

1401     }

1402   }

1403   cgraph_remove_unreachable_nodes ();

1404

1405   if (cgraph_dump_file )

1406     fprintf (cgraph_dump_file ,

1407            "/nInlined %i calls, eliminated %i functions, "

1408             "%i insns turned to %i insns./n/n",

1409            ncalls_inlined , nfunctions_inlined , initial_insns ,

1410            overall_insns );

1411   free (order);

1412   free (inlined);

1413   free (inlined_callees);

1414 }

 

Back cgraph_decide_inline , for small function only be called once, and is wrong to kill above, the compiler still gives a way out, after all the benefit of inlining these small functions is relatively large. At line 1403, it does accessibility analysis to remove unreachable functions.

5.13.5.3.2.3.  Set cgraph_global_info_ready

Return to cgraph_optimize , in above processing global field of cgraph_node nodes have been set, at line 1596, set cgraph_global_info_ready to show this fact.