压缩C++简单程序词法分析后的文件（编译原理实验）

  继续完成前面一篇“设计有穷自动机DFA实现C++简单程序的词法分析、扫描（编译原理实验）”词法分析扫猫程序剩下来关于去除多余空行、空格、注释进行源程序压缩的功能。

按实验要求（如下），这里需要考虑下面带星号*的第（3）（5）点:

 

实验中用到的C++源程序如下图：

 

思路：

其实也就是将源程序中的多余空格、注释、换行等都删除，整理成单单一行的源代码。
每次对扫描程序获取到的Token进行判断，根据上一个Token的类型（有关键字、标识符、数值、字符串、特殊符号）决定当前Token是否能够与上一个Token紧邻，也即不加任何空格。
例如上面截图中倒数第二行中的 else 和 cout 两个关键字之间就必须有空格分开，否则代码就会出错了。针对上面这个简单的C++源程序，观察其DFA图可以得出以下特点：
1、关键字与标识符不能紧邻，例如 int i中间必须有空格
2、关键字与关键字也不能紧邻，如上所述
3、另外关键字与字符串也不要紧邻

对于以上样例输入，先进行词法分析，然后将获得的Token压缩并保存在StringBuilder对象中，在写入到一个新的文件，最终再次对压缩后的文件进行扫描，判断压缩前后的扫描结果是否一直。

程序输出结果（包括压缩后的源代码）如下：

 

根据上面这三个特点，代码实现如下（高亮部分是与上一篇源代码不同之处）：

package lexical_analysis;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.PrintWriter;

public class Scanner_2 {

    // 定义DFA中的所有状态表
//  enum StateType {Start, Num, ID, EQ, NE, NM, NL,
//      Com, LineCom, MulCom1, MulCom2, Special, Done, Str};

    // 定义DFA中的所有状态表
    private static final int Start = 1;
    private static final int Num = 2;
    private static final int ID = 3;
    private static final int EQ = 4;
    private static final int NE = 5;
    private static final int NM = 6;
    private static final int NL = 7;
    private static final int Coms = 8;
    private static final int LineCom = 9;
    private static final int MulCom1 = 10;
    private static final int MulCom2 = 11;
    private static final int Special = 12;
    private static final int Done = 13;
    private static final int Str = 14;

    // Token类型，Initial为初始类型
    private enum TokenType {
        Initial, ID, Special, Str, KeyWord
    };

    // 关键字
    private String[] keyWords = new String[] {
        "include", "define", "iostream", "int", "folat", "double",
        "main", "if", "else", "for", "while", "do", "goto", "switch",
        "case", "static", "cin", "cout"
    };

    // 特殊字符
    private String [] special = {"{", "}", "[", "]", "(", ")",
             "#", ",", ".", ";", ":", "\\",
             "'", "\"", ">>", "<<", "!=", "=",
             "==", "<=", ">=", "++", "--"};

    // 算术运算符
    private String [] arithmetic = {"+", "-", "-", "/", "%"};

    // 源代码文件输入流
    private BufferedReader sourceFile;

    // 压缩后的文件输出流
    private PrintWriter compressedFileWriter;
    // 上一个Token的类型
    private TokenType preType = TokenType.Initial;
    // 缓存去除多余空格、注释后的源代码
    private StringBuilder compressedStr = new StringBuilder();

    // 扫描行的最大字符数
    private static final int BUF_SIZE = 256;
    // 当前行的字符长度
    private int bufSize = 0;
    // 当前行
    private String eachLine;
    // 当前扫描行的字符序列
    private char [] lineBuf = new char[BUF_SIZE];
    // 当前扫描的行数
    private int lineNum = 0;
    // 当前行的字符下标
    private int charPos = 0;
    // 是否已达文件尾
    private boolean isEOF = false;

    /**
     * 每次扫描前都要初始化一些必要变量值
     */
    private void initial(){
        bufSize = 0;
        lineNum = 0;
        charPos = 0;
        isEOF = false;
    }

    /**
     * 初始化并读取源代码文件
     * 扫描程序开始执行，直到读取文件结束符EOF
     * @throws Exception
     */
    private void scanning(String originalFile) throws Exception {
        this.sourceFile = new BufferedReader(new FileReader(originalFile));

        this.initial();
        while(!isEOF) {
            getToken();
        }
        System.out.println("========================> end scanning ...");
    }

    /**
     * 获取下一个字符
     * @return
     * @throws Exception
     */
    private char getNextChar() throws Exception {
        char nextChar = '\0';

        if(!(charPos < bufSize)) {
            if((eachLine = sourceFile.readLine()) != null) {
                lineNum++;
                System.out.println(lineNum + ": " + eachLine);
                lineBuf = eachLine.toCharArray();
                bufSize = eachLine.length();
                charPos = 0;
                nextChar = lineBuf[charPos++];
            } else {
                isEOF = true;
                nextChar = '\0';
            }
        } else {
            nextChar = lineBuf[charPos++];
        }
        return nextChar;
    }

    /**
     * 【按步长（step）】取消获取下一个字符
     */
    private void unGetNextChar(int step) {
        if(!isEOF) {
            charPos -= step;
        }
    }

    /**
     * 获取一个Token
     * @return
     * @throws Exception
     */
    private String getToken() throws Exception {
        String tokenStr = "";
        String currentToken = "";
        int currentState = Start;
        boolean isSave;

        // 不同时为EOF和Done状态
        while(currentState != Done && !isEOF) {
            char c = getNextChar();
            isSave = true;

            switch(currentState) {
                case Start:
                    if(isDigit(c)) {
                        currentState = Num;
                    } else if(isLetter(c) || c == '.') { //点号是为了处理头文件iostream.h的格式
                        currentState = ID;
                    } else if(c == ' ' || c == '\t' || c == '\n') {
                        isSave = false;
                    } else if(c == '!') {
                        currentState = NE;
                    } else if(c == '=') {
                        currentState = EQ;
                    } else if(c == '<') {
                        currentState = NM;
                    } else if(c == '>') {
                        currentState = NL;
                    } else if(c == '/') {
                        currentState = Coms;
                        isSave = false;
                    } else if(c == '"') {
                        currentState = Str;
                    } else {
                        currentState = Done;
//                      if(isSingle(c)) {
//                          currentToken = "" + c;
//                          currentState = Done;
//                          isSave = false;
//                      }
                    }
                    break;
                case Num:
                    if(!isDigit(c)) {
                        currentState = Done;
                        unGetNextChar(1);
                        isSave = false;
                    }
                    break;
                case ID:
                    if(!isLetter(c) && !isDigit(c)) {
                        currentState = Done;
                        unGetNextChar(1);
                        isSave = false;
                    }
                    break;
                case NE:
                    if(c != '=') {
                        currentState = Special;
                        unGetNextChar(2);
                        isSave = false;
                    } else {
                        currentState = Done;
                    }
                    break;
                case NM:
                    if(c != '=' && c != '<') {
                        currentState = Special;
                        unGetNextChar(2);
                        isSave = false;
                    } else {
                        currentState = Done;
                    }
                    break;
                case NL:
                    if(c != '=' && c != '>') {
                        currentState = Special;
                        unGetNextChar(2);
                        isSave = false;
                    } else {
                        currentState = Done;
                    }
                    break;
                case EQ:
                    if(c != '=') {
                        currentState = Special;
                        unGetNextChar(2);
                        isSave = false;
                    } else {
                        currentState = Done;
                    }
                    break;
                case Str:
                    if(c == '"') {
                        currentState = Done;
                    }
                    break;
                case Coms:
                    isSave = false;
                    if(c == '/') {
                        currentState = LineCom;
                    } else if(c == '*') {
                        currentState = MulCom1;
                    } else {
                        currentState = Special;
                        unGetNextChar(1);
                    }
                    break;
                case LineCom:
                    isSave = false;
                    if(c == '\n') {
                        currentState = Done;
                    }
                    break;
                case MulCom2:
                    isSave = false;
                    if(c == '*') {
                        currentState = MulCom2;
                    } else if(c == '/') {
                        currentState = Done;
                    } else {
                        currentState = MulCom1;
                    }
                    break;
                case Special:
                    if(c == '!' || c == '=' || c == '<' || c == '>') {
//                  if(isSpecialSingle(c)) {
                        currentToken = "" + c;
                        currentState = Done;
                        isSave = false;
                    } else {
                        currentToken = "Error";
                        currentState = Done;
                    }
                    break;
                default:
                    System.out.println(lineNum + " >> Scanner Bug : state = " + currentState);
                    currentState = Done;
                    currentToken = "Error";
                    break;
            }
            if(isSave) {
                tokenStr += c;
            }
            if(currentState == Done) {
                currentToken = tokenStr;
                printToken(currentToken);
            }
        }
        return currentToken;
    }

    /**
     * 判断是否为字母
     * @param c
     * @return
     */
    private boolean isLetter(char c) {
        if(('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z')){
            return true;
        }
        return false;
    }

    /**
     * 判断是否为数字
     * @param c
     * @return
     */
    private boolean isDigit(char c) {
        if('0' <= c && c <= '9') {
            return true;
        }
        return false;
    }

    /**
     * 打印时判断是否为【数值Num】
     * @param token
     * @return
     */
    private boolean isNum(String token) {
        boolean flag = true;
        char [] chs = token.toCharArray();
        int len = chs.length;
        for(int i = 0; i < len; i++) {
            if(!isDigit(chs[i])) {
                flag = false;
            }
        }
        return flag;
    }

    /**
     * 打印时判断是否为【特殊符号】
     */
    private boolean isSpecial(String token) {
        int len = special.length;
        for(int i = 0; i < len; i++) {
            if(token.equals(special[i])) {
                return true;
            }
        }
        return false;
    }

    /**
     * 判断是否为算术运算符
     * @param token
     * @return
     */
    private boolean isArithmetic(String token) {
        int len = arithmetic.length;
        for(int i = 0; i < len; i++) {
            if(token.equals(arithmetic[i])) {
                return true;
            }
        }
        return false;
    }

    /**
     * 打印时判断是否为【关键字】
     * @param token
     * @return
     */
    private boolean isKeyWord(String token) {
        int len = keyWords.length;
        for(int i = 0; i < len; i++) {
            if(keyWords[i].equals(token)) {
                return true;
            }
        }
        return false;
    }

    /**
     * 判断是否为【单个字符】即 # * { } [ ] ( ) , . ; : '
     * @param c
     * @return
     */
//  private boolean isSingle(char c) {
//      char [] single = {'#', '*', '{', '}',
//                          '[', ']', '(', ')',
//                          ':', ';', '.', ',',
//                          '\''};
//      int len = single.length;
//      for(int i = 0; i < len; i++) {
//          if(c == single[i]) {
//              return true;
//          }
//      }
//      return false;
//  }

    /**
     * 判断是否为【单个的特殊字符】即 !   =   <   >
     * 因为这几个属于多义字符，能形成 !=  ==  <<  >>
     * @param c
     * @return
     */
//  private boolean isSpecialSingle(char c) {
//      char [] special = {'!', '=', '<', '>'};
//      int len = special.length;
//      for(int i = 0; i < len; i++) {
//          if(c == special[i]) {
//              return true;
//          }
//      }
//      return false;
//  }

    /**
     * 按类别打印扫描得到的Token
     * @param token
     */
    private void printToken(String token) {
        if(isKeyWord(token)) {
            System.out.printf("%4d: %s --- %s\n", lineNum, token, "关键字");

            token = (preType == TokenType.KeyWord ? " " : "") + token;
            preType = TokenType.KeyWord;
            this.compressedStr.append(token);

        } else if(isSpecial(token)) {
            System.out.printf("%4d: %s --- %s\n", lineNum, token,"特殊符号");

            preType = TokenType.Special;
            this.compressedStr.append(token);

        } else if(isArithmetic(token)) {
            System.out.printf("%4d: %s --- %s\n", lineNum, token,"算术运算符");

            preType = TokenType.Special;
            this.compressedStr.append(token);

        } else if(isNum(token)) {
            System.out.printf("%4d: %s --- %s\n", lineNum, token,"数值");

            preType = TokenType.Special;
            this.compressedStr.append(token);

        } else if(token.startsWith("\"")) {
            System.out.printf("%4d: %s --- %s\n", lineNum, token,"字符串");

            token = (preType == TokenType.KeyWord ? " " : "") + token;
            this.compressedStr.append(token);
            preType = TokenType.Str;

        } else {
            System.out.printf("%4d: %s --- %s\n", lineNum, token,"标识符");

            token = (preType == TokenType.KeyWord ? " " : "") + token;
            this.compressedStr.append(token);
            preType = TokenType.ID;
        }
    }

    /**
     * 打印并将被压缩后的源代码写入新的文件中
     */
    public void printCompressedFile(String compressedFile) throws Exception {
        System.out.println(this.compressedStr);
        // 创建压缩后的文件输出流
        this.compressedFileWriter = new PrintWriter(
                new FileOutputStream(new File(compressedFile)));
        // 写入到新的文件
        this.compressedFileWriter.write(new String(this.compressedStr));
        this.compressedFileWriter.flush();
    }

    /**
     * 测试
     */
    public static void main(String[] args) throws Exception {
        Scanner_2 scanner = new Scanner_2();

        System.out.println("扫描未压缩源代码文件 >> ");
        scanner.scanning("cppSrc.cpp");

        System.out.println("\n压缩之后的源代码 >> ");
        scanner.printCompressedFile("afterCompressed.cpp");

        System.out.println("\n扫描压缩后的源代码文件 >> ");
        scanner.scanning("afterCompressed.cpp");
    }
}