tokenize method

Implementation

List<Token> tokenize(String inputString) {
  final List<Token> tempTokenStream = <Token>[];
  final String clearedString = inputString.replaceAll(' ', '').trim();
  final RuneIterator iter = clearedString.runes.iterator;

  while (iter.moveNext()) {
    final String si = iter.currentAsString;
    /*
     * Check if the current Character is a keyword. If it is a keyword, check if the intBuffer is not empty and add
     * a Value Token for the intBuffer and the corresponding Token for the keyword.
     */
    bool keywordsContainsKey = keywords.containsKey(si);

    /*
    * There's a situation that 'ceil' conflict with 'e', we use this to look back the buffer and decide
    * which way should go.
    */
    if (si == 'e' && varBuffer.isNotEmpty) {
      keywordsContainsKey = false;
    }

    if (keywordsContainsKey) {
      // check and or do intBuffer and varBuffer
      if (intBuffer.isNotEmpty) {
        _doIntBuffer(tempTokenStream);
      }
      if (varBuffer.isNotEmpty) {
        _doVarBuffer(tempTokenStream);
      }
      // MH - Bit of a hack here to handle exponentials of the form e^x rather than e(x)
      if (keywords[si] == TokenType.POW &&
          tempTokenStream.last.type == TokenType.EFUNC) {
        // Clear varBuffer since we have nothing to add to the stream as EFUNC is already in it
        //_doVarBuffer(tempTokenStream);
        varBuffer = '';
      } else {
        // Normal behaviour
        tempTokenStream.add(Token(si, keywords[si]!));
      }
    } else {
      // Check if the current string is a Number. If it's the case add the string to the intBuffer.
      StringBuffer sb = StringBuffer(intBuffer);
      try {
        // A variable can contains digits
        if (varBuffer.isNotEmpty) {
          throw FormatException();
        }
        int.parse(si);
        // The current string is a number and it is added to the intBuffer.
        sb.write(si);
        intBuffer = sb.toString();
        if (varBuffer.isNotEmpty) {
          _doVarBuffer(tempTokenStream);
        }
      } on FormatException {
        // Check if the current string is part of a floating point input
        if (si == '.') {
          sb.write(si);
          intBuffer = sb.toString();
          continue;
        }

        // The current string is not a number and not a simple keyword, so it has to be a variable or function.
        sb = StringBuffer(varBuffer);
        if (intBuffer.isNotEmpty) {
          /*
           * The intBuffer contains a string and the current string is a
           * variable or part of a complex keyword, so the value is added
           * to the token stream and the current string is added to the
           * var buffer.
           */
          _doIntBuffer(tempTokenStream);
          sb.write(si);
          varBuffer = sb.toString();
        } else {
          // intBuffer contains no string and the current string is a variable, so both Tokens are added to the tokenStream.
          sb.write(si);
          varBuffer = sb.toString();
        }
      }
    }
  }

  if (intBuffer.isNotEmpty) {
    // There are no more symbols in the input string but there is still an int in the intBuffer
    _doIntBuffer(tempTokenStream);
  }
  if (varBuffer.isNotEmpty) {
    // There are no more symbols in the input string but there is still a variable or keyword in the varBuffer
    _doVarBuffer(tempTokenStream);
  }
  return tempTokenStream;
}