EXPERIMENT- 2
Lexical Analysis and Symbol Table Generation
M SANGEETHGOWTHAM
22BLC1386
AIM
Implement a lexical analyzer capable of tokenizing a simple programming language with
constructs like keywords, numbers, identifiers, operators, punctuators, and boolean
literals.
Classify tokens based on their types such as:
● Keywords
● Numbers (integer values)
● Identifiers (variables and function names)
● Operators
● Punctuators
● Boolean literals
DESIGN CONSTRAINTS
Implement a lexical analyzer capable of tokenizing a simple programming language with
constructs like keywords, numbers, identifiers, operators, punctuators, and boolean
literals.
Classify tokens based on their types such as:
● Keywords (e.g., if, else, int, return)
● Numbers (integer values)
● Identifiers (variables and function names)
● Operators (e.g., +, -, =)
● Punctuators (e.g., ;)
● Boolean literals (e.g., true, false)
Output Format:
● The tokenized representation of each statement will be printed.
● The symbol table will be displayed at the end of the process, listing all unique
identifiers encountered with their respective types.
● A validation message (Valid statement. or Invalid statement.) will be
shown for each processed statement.
Assumptions:
● The language syntax is simplified for the purposes of this lab and does not
include complex constructs like loops, conditionals, or functions.
● Only simple assignments are considered valid statements for this project.
Constraints on Token Length:
● Identifiers are assumed to be no longer than 50 characters.
● Numbers are assumed to be positive integers (no floating-point numbers or
negative values).
● Only single-character operators and basic punctuators (such as ;) are
considered.
CODE
#include <iostream>
#include <fstream>
#include <sstream>
#include <vector>
#include <map>
#include <string>
#include <stdexcept>
#include <cctype>
#include <iomanip>
using namespace std;
struct SymbolInfo {
string type;
double value;
};
struct LexicalEntry {
string name;
string type;
string category;
};
void vectorise(string st, char split, vector<vector<string>>&
compiled_stream) {
stringstream ss(st);
vector<string> split_vector;
string token;
while (getline(ss, token, split)) {
if(!token.empty()){
split_vector.push_back(token);
compiled_stream.push_back(split_vector);
bool isNumeric(const string& str) {
if (str.empty()) return false;
bool hasDecimal = false;
for (char c : str) {
if (!isdigit(c) && c != '-' && c != '.') {
return false;
if (c == '.') {
if (hasDecimal) {
return false;
hasDecimal = true;
}
return true;
double evaluateExpression(const string& expression, const
map<string, SymbolInfo>& symbolTable) {
if (isNumeric(expression)) {
return stod(expression);
} else if (symbolTable.count(expression)) {
return symbolTable.at(expression).value;
} else {
throw runtime_error("Invalid expression: " + expression);
void generateLexicalTable(const vector<vector<string>>&
compiled_stream, map<int, LexicalEntry>& lexicalTable) {
int nextId = 1;
// Keywords
lexicalTable[nextId++] = {"int", "Keyword", "Reserved"};
lexicalTable[nextId++] = {"float", "Keyword", "Reserved"};
//Operators
lexicalTable[nextId++] = {"=", "Operator", "Symbol"};
lexicalTable[nextId++] = {"+", "Operator", "Symbol"};
lexicalTable[nextId++] = {"-", "Operator", "Symbol"};
for (const auto& tokens : compiled_stream) {
for (const auto& token: tokens) {
bool found = false;
for (const auto& pair : lexicalTable){
if(pair.second.name == token){
found = true;
break;
if (!found) {
if (isNumeric(token)) {
lexicalTable[nextId++] = {token, "Literal", "Constant"};
} else {
lexicalTable[nextId++] = {token, "Identifier", "Variable"};
}
void processTokens(const vector<vector<string>>&
compiled_stream, const map<int, LexicalEntry>& lexicalTable,
map<string, SymbolInfo>& symbolTable, vector<string>&
tokenStream) {
for (const auto& tokens : compiled_stream) {
string lineTokenStream;
for(const auto& token : tokens){
for (const auto& pair : lexicalTable){
if (pair.second.name == token){
if(pair.second.type == "Keyword"){
lineTokenStream += "<" + to_string(pair.first) + ">";
else if(pair.second.type == "Identifier"){
lineTokenStream += "<id" + to_string(pair.first) + ">";
else if(pair.second.type == "Operator"){
lineTokenStream += "<op" + to_string(pair.first) +
">";
else if (pair.second.type == "Literal"){
lineTokenStream += "<" + to_string(pair.first) + ">";
break;
tokenStream.push_back(lineTokenStream);
if (tokens.size() >= 3) {
if ((tokens[0] == "int" || tokens[0] == "float") && tokens[2] ==
"=" ) {
string type = tokens[0];
string name = tokens[1];
string value_str = tokens[3];
if (symbolTable.count(name)) {
throw runtime_error("Variable " + name + " already
declared.");
} else if (!isNumeric(value_str)) {
throw runtime_error("Initial value must be an integer or
float: " + value_str);
} else {
double value = stod(value_str);
symbolTable[name] = { type, value };
} else if(tokens.size() >= 3 && tokens[1] == "="){
string name = tokens[0];
if(symbolTable.count(name)){
string expression = tokens[2];
try {
double newValue = evaluateExpression(expression,
symbolTable);
symbolTable[name].value = newValue;
} catch(const runtime_error& err){
throw runtime_error("Error calculating the value: " +
std::string(err.what()) );
} else {
throw runtime_error("Variable " + name + " not declared
before being used.");
} else {
if (!tokens.empty()) {
throw runtime_error("Invalid syntax found near token " +
tokens[0] + ".");
int main() {
vector<vector<string>> compiled_stream;
string txt;
ifstream MyReadFile("tokenise.txt");
if (!MyReadFile.is_open()) {
cerr << "Error: Could not open file tokenise.txt\n";
return 1;
}
while (getline(MyReadFile, txt)) {
vectorise(txt, ' ', compiled_stream);
map<int, LexicalEntry> lexicalTable;
generateLexicalTable(compiled_stream, lexicalTable);
cout << "Lexical Table:\n";
cout << setw(5) << "ID" << setw(10) << "Name" << setw(15) <<
"Type" << setw(15) << "Category" << endl;
for (const auto& pair : lexicalTable) {
cout << setw(5) << pair.first
<< setw(10) << pair.second.name
<< setw(15) << pair.second.type
<< setw(15) << pair.second.category << endl;
cout << endl;
map<string, SymbolInfo> symbolTable;
vector<string> tokenStream;
try{
processTokens(compiled_stream, lexicalTable, symbolTable,
tokenStream);
} catch(const runtime_error& error){
cerr << "Error: " << error.what() << endl;
return 1;
cout << "Symbol Table:\n";
cout << "Id_name\tType\tValue\n";
for (const auto& pair : symbolTable) {
cout << pair.first << "\t" << pair.second.type << "\t" <<
pair.second.value << endl;
cout << endl;
cout << "Token Stream (Line by Line):\n";
for(const auto& line: tokenStream){
cout << line << endl;
}
cout << endl;
return 0;
INPUT FILE
OUTPUT
