Studying note of GCC-3.4.6 source (37)

4.1.3.1.2.2.      Read in common file

Back should_stack_file, for common file, the whole file content will be put into buffer of cpp_reader.

 

should_stack_file (continue)

 

584      if (!read_file (pfile, file))

585        return false;

586   

587      /* Now we've read the file's contents, we can stack it if there

588        are no once-only files.  */

589      if (!pfile->seen_once_only)

590        return true;

 

For common file, reading in its content is quite straight-forward.

 

523    static bool

524    read_file (cpp_reader *pfile, _cpp_file *file)                                              in cppfiles.c

525    {

526      /* If we already have its contents in memory, succeed immediately.  */

527      if (file->buffer_valid)

528        return true;

529   

530      /* If an earlier read failed for some reason don't try again.  */

531      if (file->dont_read || file->err_no)

532        return false;

533   

534      if (file->fd == -1 && !open_file (file))

535      {

536        open_file_failed (pfile, file, 0);

537        return false;

538      }

539   

540      file->dont_read = !read_file_guts (pfile, file);

541      close (file->fd);

542      file->fd = -1;

543   

544      return !file->dont_read;

545    }

 

Nonzero dont_read field in cpp_reader above indicates if file has read failed. For that file don’t attempt to open it anymore.

 

450    static bool

451    read_file_guts (cpp_reader *pfile, _cpp_file *file)                                      in cppfiles.c

452    {

453      ssize_t size, total, count;

454      uchar *buf;

455      bool regular;

456     

457      if (S_ISBLK (file->st.st_mode))

458      {

459        cpp_error (pfile, CPP_DL_ERROR, "%s is a block device", file->path);

460        return false;

461      }

462   

463      regular = S_ISREG (file->st.st_mode);

464      if (regular)

465      {

466        /* off_t might have a wider range than ssize_t - in other words,

467          the max size of a file might be bigger than the address

468          space. We can't handle a file that large. (Anyone with

469          a single source file bigger than 2GB needs to rethink

470          their coding style.) Some systems (e.g. AIX 4.1) define

471          SSIZE_MAX to be much smaller than the actual range of the

472          type. Use INTTYPE_MAXIMUM unconditionally to ensure this

473          does not bite us.  */

474        if (file->st.st_size > INTTYPE_MAXIMUM (ssize_t))

475        {

476          cpp_error (pfile, CPP_DL_ERROR, "%s is too large", file->path);

477          return false;

478        }

479   

480        size = file->st.st_size;

481      }

482      else

483        /* 8 kilobytes is a sensible starting size. It ought to be bigger

484          than the kernel pipe buffer, and it's definitely bigger than

485          the majority of C source files.  */

486        size = 8 * 1024;

487   

488      buf = xmalloc (size + 1);

489      total = 0;

490      while ((count = read (file->fd, buf + total, size - total)) > 0)

491      {

492        total += count;

493   

494        if (total == size)

495        {

496          if (regular)

497            break;

498          size *= 2;

499          buf = xrealloc (buf, size + 1);

500        }

501      }

502   

503      if (count < 0)

504      {

505        cpp_errno (pfile, CPP_DL_ERROR, file->path);

506        return false;

507      }

508   

509      if (regular && total != size && STAT_SIZE_RELIABLE (file->st))

510        cpp_error (pfile, CPP_DL_WARNING,

511                 "%s is shorter than expected", file->path);

512   

513      file->buffer = _cpp_convert_input (pfile, CPP_OPTION (pfile, input_charset),

514                                   buf, size, total, &file->st.st_size);

515      file->buffer_valid = true;

516   

517      return true;

518    }

 

Above _cpp_convert_input is used to convert the content if the file using charset other than UTF-8 or UTF-EBCDIC by using system call iconv. For the detail, you can refer to souce, cppcharset.c, strcasecmp.c.

If the file is not included by #pragma once or #import, after reading in the content, it exits should_stack_file at line 590. Otherwise, it should ensure only one copy of content is available.

 

should_stack_file (continue)

 

592      /* We may have read the file under a different name. Look

593        for likely candidates and compare file contents to be sure.  */

594      for (f = pfile->all_files; f; f = f->next_file)

595      {

596        if (f == file)

597          continue;

598   

599        if ((import || f->once_only)

600           && f->err_no == 0

601           && f->st.st_mtime == file->st.st_mtime

602           && f->st.st_size == file->st.st_size)

603        {

604          _cpp_file *ref_file;

605          bool same_file_p = false;

606   

607          if (f->buffer && !f->buffer_valid)

608          {

609            /* We already have a buffer but it is not valid, because

610              the file is still stacked. Make a new one.  */

611             ref_file = make_cpp_file (pfile, f->dir, f->name);

612            ref_file->path = f->path;

613          }

614          else

615            /* The file is not stacked anymore. We can reuse it.  */

616            ref_file = f;

617   

618          same_file_p = read_file (pfile, ref_file)

619             /* Size might have changed in read_file().  */

620             && ref_file->st.st_size == file->st.st_size

621             && !memcmp (ref_file->buffer,

622                                  file->buffer,

623                                  file->st.st_size);

624   

625          if (f->buffer && !f->buffer_valid)

626          {

627            ref_file->path = 0;

628            destroy_cpp_file (ref_file);

629          }

630   

631          if (same_file_p)

632            break;

633        }

634      }

635   

636      return f == NULL;

637    }

 

Above at line 594, all_files in cpp_reader saves all already read in files belonging to current compilation unit; while at line 622, file is the file just read in. Then at line 636, if is NULL, it means file doesn’t exists in all_files in cpp_reader, which should be stacked.

4.1.3.1.2.3.      Stack the file (exclude PCH file)

If should_stack_file returns true, the file needs to be stacked (see PCH file will cause the function returns false, it will not be stacked). At line 672 and 673 below, mi_valid and mi_cmacro are used for multiple include optimization, the detail of the technique is given below from [5]. We will see the steps in source code in later section:

Header files are often of the form      #ifndef FOO      #define FOO      ...      #endif to prevent the compiler from processing them more than once. The preprocessor notices such header files, so that if the header file appears in a subsequent `#include' directive and `FOO' is defined, then it is ignored and it doesn't preprocess or even re-open the file a second time. This is referred to as the "multiple include optimization". Under what circumstances is such an optimization valid? If the file were included a second time, it can only be optimized away if that inclusion would result in no tokens to return, and no relevant directives to process. Therefore the current implementation imposes requirements and makes some allowances as follows: Ÿ        There must be no tokens outside the controlling `#if'-`#endif' pair, but whitespace and comments are permitted. Ÿ        There must be no directives outside the controlling directive pair, but the "null directive" (a line containing nothing other than a single `#' and possibly whitespace) is permitted. Ÿ        The opening directive must be of the form           #ifndef FOO      or           #if !defined FOO     [equivalently, #if !defined(FOO)] Ÿ        In the second form above, the tokens forming the `#if' expression must have come directly from the source file--no macro expansion must have been involved. This is because macro definitions can change, and tracking whether or not a relevant change has been made is not worth the implementation cost. Ÿ        There can be no `#else' or `#elif' directives at the outer conditional block level, because they would probably contain something of interest to a subsequent pass. First, when pushing a new file on the buffer stack, _stack_include_file sets the controlling macro mi_cmacro to `NULL', and sets mi_valid to `true'. This indicates that the preprocessor has not yet encountered anything that would invalidate the multiple-include optimization. As described in the next few paragraphs, these two variables having these values effectively indicates top-of-file. When about to return a token that is not part of a directive, _cpp_lex_token sets mi_valid to `false'. This enforces the constraint that tokens outside the controlling conditional block invalidate the optimization. The do_if, when appropriate, and do_ifndef directive handlers pass the controlling macro to the function push_conditional. Cpplib maintains a stack of nested conditional blocks, and after processing every opening conditional this function pushes an if_stack structure onto the stack. In this structure it records the controlling macro for the block, provided there is one and we're at top-of-file (as described above). If an `#elif' or `#else' directive is encountered, the controlling macro for that block is cleared to `NULL'. Otherwise, it survives until the `#endif' closing the block, upon which do_endif sets mi_valid to true and stores the controlling macro in mi_cmacro. _cpp_handle_directive clears mi_valid when processing any directive other than an opening conditional and the null directive. With this, and requiring top-of-file to record a controlling macro, and no `#else' or `#elif' for it to survive and be copied to mi_cmacro by do_endif, we have enforced the absence of directives outside the main conditional block for the optimization to be on. Note that whilst we are inside the conditional block, mi_valid is likely to be reset to `false', but this does not matter since the the closing `#endif' restores it to `true' if appropriate. Finally, since _cpp_lex_direct pops the file off the buffer stack at `EOF' without returning a token, if the `#endif' directive was not followed by any tokens, mi_valid is `true' and _cpp_pop_file_buffer remembers the controlling macro associated with the file. Subsequent calls to _stack_include_file result in no buffer being pushed if the controlling macro is defined, effecting the optimization. A quick word on how we handle the case.      #if !defined FOO _cpp_parse_expr and parse_defined take steps to see whether the three stages `!', `defined-expression' and `end-of-directive' occur in order in a `#if' expression. If so, they return the guard macro to do_if in the variable mi_ind_cmacro, and otherwise set it to `NULL'. enter_macro_context sets mi_valid to false, so if a macro was expanded whilst parsing any part of the expression, then the top-of-file test in push_conditional fails and the optimization is turned off.

 

_cpp_stack_file (continue)

 

652      sysp = MAX ((pfile->map ? pfile->map->sysp : 0),

653               (file->dir ? file->dir->sysp : 0));

654   

655      /* Add the file to the dependencies on its first inclusion.  */

656      if (CPP_OPTION (pfile, deps.style) > !!sysp && !file->stack_count)

657      {

658        if (!file->main_file || !CPP_OPTION (pfile, deps.ignore_main_file))

659          deps_add_dep (pfile->deps, file->path);

660      }

661   

662      /* Clear buffer_valid since _cpp_clean_line messes it up.  */

663      file->buffer_valid = false;

664      file->stack_count++;

665   

666      /* Stack the buffer.  */

667      buffer = cpp_push_buffer (pfile, file->buffer, file->st.st_size,

668                           CPP_OPTION (pfile, preprocessed));

669      buffer->file = file;

670   

671      /* Initialize controlling macro state.  */

672      pfile->mi_valid = true;

673      pfile->mi_cmacro = 0;

674   

675      /* Generate the call back.  */

676      _cpp_do_file_change (pfile, LC_ENTER, file->path, 1, sysp);

677   

678      return true;

679    }

4.1.3.1.2.3.1.              Do file change

When read the file (non-PCH file) into the buffer, lexer (it also provides some functionalities of preprocessor, like macro expansion), which output will be syntax analyzed by parser. But before switching buffers, there are some processings needed.

 

906    void

907    _cpp_do_file_change (cpp_reader *pfile, enum lc_reason reason,                  in cpplib.c

908                     const char *to_file, unsigned int file_line,

909                     unsigned int sysp)

910    {

911       pfile->map = linemap_add (&pfile->line_maps, reason, sysp,

912                           pfile->line, to_file, file_line);

913   

914      if (pfile->cb.file_change)

915        pfile->cb.file_change (pfile, pfile->map);

916    }

 

Notice that line in pfile above records the current position in the file in line number, and parameter set in linemap_add refers to line_maps of pfile. Then parameters to_file and to_line mention to the included file (if the file is introduced by #include) or the file itself. At here, as we first open the interested file, to_line is 1. Slot used in line_maps indicates how many files have been line mapped within this file context, and field depth indicates the position in the include list.

 

76      const struct line_map *

77      linemap_add (struct line_maps *set, enum lc_reason reason,                        in line-map.c

78                unsigned int sysp, source_location from_line,

79                const char *to_file, unsigned int to_line)

80      {

81        struct line_map *map;

82     

83        if (set->used && from_line < set->maps[set->used - 1].from_line)

84          abort ();

85     

86        if (set->used == set->allocated)

87        {

88          set->allocated = 2 * set->allocated + 256;

89          set->maps = xrealloc (set->maps, set->allocated * sizeof (struct line_map));

90        }

91     

92        map = &set->maps[set->used++];

93     

94        if (to_file && *to_file == '/0')

95          to_file = "<stdin>";

96     

97        /* If we don't keep our line maps consistent, we can easily

98          segfault. Don't rely on the client to do it for us.  */

99        if (set->depth == 0)

100        reason = LC_ENTER;

101      else if (reason == LC_LEAVE)

102      {

103        struct line_map *from;

104        bool error;

105   

106        if (MAIN_FILE_P (map - 1))

107        {

108          if (to_file == NULL)

109          {

110             set->depth--;

111             set->used--;

112             return NULL;

113           }

114           error = true;

115           reason = LC_RENAME;

116           from = map - 1;

117         }

118         else

119         {

120          from = INCLUDED_FROM (set, map - 1);

121          error = to_file && strcmp (from->to_file, to_file);

122        }

123   

124        /* Depending upon whether we are handling preprocessed input or

125          not, this can be a user error or an ICE.  */

126        if (error)

127          fprintf (stderr, "line-map.c: file /"%s/" left but not entered/n",

128                to_file);

129   

130        /* A TO_FILE of NULL is special - we use the natural values.  */

131        if (error || to_file == NULL)

132        {

133          to_file = from->to_file;

134          to_line = LAST_SOURCE_LINE (from) + 1;

135          sysp = from->sysp;

136        }

137      }

138   

139      map->reason = reason;

140      map->sysp = sysp;

141      map->from_line = from_line;

142      map->to_file = to_file;

143      map->to_line = to_line;

144   

145      if (reason == LC_ENTER)

146      {

147        map->included_from = set->depth == 0 ? -1 : (int) (set->used - 2);

148        set->depth++;

149        if (set->trace_includes)

150          trace_include (set, map);

151      }

152      else if (reason == LC_RENAME)

153        map->included_from = map[-1].included_from;

154      else if (reason == LC_LEAVE)

155      {

156        set->depth--;

157        map->included_from = INCLUDED_FROM (set, map - 1)->included_from;

158      }

159   

160      return map;

161    }

 

See that at line 106 above, (map-1) is used as the parameter in MAIN_FILE_P.

 

120    #define MAIN_FILE_P(MAP) ((MAP)->included_from < 0)                             in line-map.h

 

And at line 120 INCLUDED_FROM is defined as:

 

117     #define INCLUDED_FROM(SET, MAP) (&(SET)->maps[(MAP)->included_from])

 

To demonstrate how line_maps and linemap_add works, assuming that we have following file stack:

Current file ß include file1 ß include file2 ß include file3

When we are handling file3, we have line_maps as below figure.

Figure 14 linemap_add, step 1

Then if we leave file3 back into file2, we have following line_maps.

 

 

 

 Figure 15 linemap_add, step 2

Then back to file1.

Figure 16 linemap_add, step 3

And even leave current file. Notice that for the case, NULL will be returned, element at position of index 5 will be reused for the next “main file”, and 5 elements before it track the mapping relation of all related files to the preprocessed output.

Figure 17 linemap_add, step 4

At line 914 in _cpp_do_file_change, if the front-end at time of file change also has something to say, it needs set hook file_change in cb structure in cpp_reader. The C++ front-end binds the hook with below function.

 

1500 static void

1501 cb_file_change (cpp_reader *pfile ATTRIBUTE_UNUSED,                          in c-opts.c

1502                const struct line_map *new_map)c

1503 {

1504   if (flag_preprocess_only)

1505     pp_file_change (new_map);

1506   else

1507     fe_file_change (new_map);

1508

1509   if (new_map == 0 || (new_map->reason == LC_LEAVE && MAIN_FILE_P (new_map)))

1510     push_command_line_include ();

1511 }

 

Below, macro NO_IMPLICIT_EXTERN_C is defined if the system header files support C++ as well as C. This macro inhibits the usual method of using system header files in C++, which is to pretend that the file's contents are enclosed in `extern "C" {...}'. This macro is defined in Linux. Below input_filename and input_line access line and file fields of input_location respectively.

 

203    void

204    fe_file_change (const struct line_map *new_map)                                      in c-lex.c

205    {

206      if (new_map == NULL)

207      {

208        map = NULL;

209        return;

210      }

211    

212      if (new_map->reason == LC_ENTER)

213      {

214        /* Don't stack the main buffer on the input stack;

215          we already did in compile_file.  */

216        if (map != NULL)

217        {

218          int included_at = SOURCE_LINE (new_map - 1, new_map->from_line - 1);

219   

220          input_line = included_at;

221          push_srcloc (new_map->to_file, 1);

222          (*debug_hooks->start_source_file) (included_at, new_map->to_file);

223    #ifndef NO_IMPLICIT_EXTERN_C

    …

231    #endif

232        }

233      }

234      else if (new_map->reason == LC_LEAVE)

235      {

236    #ifndef NO_IMPLICIT_EXTERN_C

    …

243    #endif

244        pop_srcloc ();

245   

246        (*debug_hooks->end_source_file) (new_map->to_line);

247      }

248   

249      update_header_times (new_map->to_file);

250      in_system_header = new_map->sysp != 0;

251      input_filename = new_map->to_file;

252      input_line = new_map->to_line;

253      map = new_map;

254   

255      /* Hook for C++.  */

256      extract_interface_info ();

257    }

 

Then for included file, push_srcloc, pop_srcloc, start_source_file, and end_source_file are invoked for debug diagnostic purpose.