#include <iostream>
#include <vector>
#include <algorithm>
#include "Utility.h"

using namespace std;


ostream& operator<<(ostream& out, const n_gram& ng){
                out << "(";
                for (int i=0; i<ng.n_-1; i++)
                        out << ng.words_[i] << ",";
                out << ng.words_[ng.n_-1]<< ")";
                return out;
        }

void Tokenize(string line, vector<string> & tokens, string delimiters){
	string token = "";
        string OneCharString = " ";
        for (int i=0; i<line.size(); i++)
                if (find(delimiters.begin(), delimiters.end(), line[i])!=delimiters.end()) // line[i] is one of the delimiter characters
                {
                        if (token != "")
                                tokens.push_back(token);
                        token = "";
                }
                else
                {
                        OneCharString[0] = line[i];
                        token +=OneCharString;
                }

        if (token != "")
                tokens.push_back(token);
}