CK.WordPredictor.TextualContextService.InternalSetRawText C# (CSharp) Method

        private void InternalSetRawText( string value )
        {
            _rawContext = value;
            _tokenCollection.Clear( false );

            if( String.IsNullOrWhiteSpace( value ) )
            {
                _tokenSeparatorIndexes = new int[0];
            }
            else
            {
                string[] tokens = Normalization( value );
                if( tokens.Length == 1 )
                {
                    _tokenCollection.Add( tokens[0] );
                }
                if( tokens.Length > 1 )
                {
                    _tokenSeparatorIndexes = new int[tokens.Length - 1];
                    _tokenSeparatorIndexes[0] = tokens[0].Length + 1;

                    for( int i = 1; i < _tokenSeparatorIndexes.Length; i++ )
                    {
                        // + 1 for whitespace
                        _tokenSeparatorIndexes[i] = _tokenSeparatorIndexes[i - 1] + 1 + tokens[i].Length; // The index of the whitespace
                    }
                    if( tokens[tokens.Length - 1] == String.Empty )
                    {
                        Array.Resize( ref tokens, tokens.Length - 1 );
                        _tokenCollection.AddRange( tokens );
                    }
                    else _tokenCollection.AddRange( tokens );
                }
            }
        }