Bamboo.Prevalence.Indexing.Tests.StringTokenizerTest.TestSimpleStrings C# (CSharp) Method

		public void TestSimpleStrings()
		{
			string text = "a foo Bar a�a\n45\n\n\n";			
			TokenAssertions.AssertTokens(new StringTokenizer(text),
				new Token("a", 0),
				new Token("foo", 2),
				new Token("Bar", 6),
				new Token("a�a", 10),
				new Token("45", 14),
				null
				);

			TokenAssertions.AssertTokens(new StringTokenizer(""),
				(Token)null);

			TokenAssertions.AssertTokens(new StringTokenizer("\n\t   "),
				(Token)null);

			TokenAssertions.AssertTokens(new StringTokenizer("\n\t  a"),
				new Token("a", 4),
				null);
		}