Lucene.Net.Analysis.Synonym.TestSlowSynonymFilter.Tokens C# (CSharp) Method

        private IList<Token> Tokens(string str)
        {
            string[] arr = str.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries);
            IList<Token> result = new List<Token>();
            for (int i = 0; i < arr.Length; i++)
            {
                string[] toks = arr[i].Split(new string[] { "/" }, StringSplitOptions.RemoveEmptyEntries);
                string[] @params = toks[0].Split(new string[] { "," }, StringSplitOptions.RemoveEmptyEntries);

                int posInc;
                int start;
                int end;

                if (@params.Length > 1)
                {
                    posInc = int.Parse(@params[1]);
                }
                else
                {
                    posInc = 1;
                }

                if (@params.Length > 2)
                {
                    start = int.Parse(@params[2]);
                }
                else
                {
                    start = 0;
                }

                if (@params.Length > 3)
                {
                    end = int.Parse(@params[3]);
                }
                else
                {
                    end = start + @params[0].Length;
                }

                Token t = new Token(@params[0], start, end, "TEST");
                t.PositionIncrement = posInc;

                result.Add(t);
                for (int j = 1; j < toks.Length; j++)
                {
                    t = new Token(toks[j], 0, 0, "TEST");
                    t.PositionIncrement = 0;
                    result.Add(t);
                }
            }
            return result;
        }