Added support for tokenization

This brings up both the `tokenizer` and `tokenSeparators` options.

dist/js/select2.amd.full.js

@@ -2646,6 +2646,93 @@ define('select2/data/tags',[
   return Tags;
 });
 
+define('select2/data/tokenizer',[
+
+], function () {
+  function Tokenizer (decorated, $element, options) {
+    var tokenizer = options.get('tokenizer');
+
+    if (tokenizer !== undefined) {
+      this.tokenizer = tokenizer;
+    }
+
+    decorated.call(this, $element, options);
+  }
+
+  Tokenizer.prototype.bind = function (decorated, container, $container) {
+    decorated.call(this, container, $container);
+
+    this.$search =  container.dropdown.$search || container.selection.$search ||
+      $container.find('.select2-search__field');
+  };
+
+  Tokenizer.prototype.query = function (decorated, params, callback) {
+    var self = this;
+
+    function select (data) {
+      self.select(data);
+    }
+
+    params.term = params.term || '';
+
+    var tokenData = this.tokenizer(params, this.options, select);
+
+    if (tokenData.term !== params.term) {
+      // Replace the search term if we have the search box
+      if (this.$search.length) {
+        this.$search.val(tokenData.term);
+        this.$search.focus();
+      }
+
+      params.term = tokenData.term;
+    }
+
+    decorated.call(this, params, callback);
+  };
+
+  Tokenizer.prototype.tokenizer = function (_, params, options, callback) {
+    var separators = options.get('tokenSeparators') || [];
+    var term = params.term;
+    var i = 0;
+
+    var createTag = this.createTag || function (params) {
+      return {
+        id: params.term,
+        text: params.term
+      };
+    };
+
+    while (i < term.length) {
+      var termChar = term[i];
+
+      if (separators.indexOf(termChar) === -1) {
+        i++;
+
+        continue;
+      }
+
+      var part = term.substr(0, i);
+      var partParams = $.extend({}, params, {
+        term: part
+      });
+
+      var data = createTag(partParams);
+
+      callback(data);
+
+      // Reset the term to not include the tokenized portion
+      term = term.substr(i + 1) || '';
+      i = 0;
+    }
+
+    return {
+      term: term
+    };
+  };
+
+  return Tokenizer;
+});
+
 define('select2/data/minimumInputLength',[
 
 ], function () {
@@ -3211,6 +3298,7 @@ define('select2/defaults',[
   './data/array',
   './data/ajax',
   './data/tags',
+  './data/tokenizer',
   './data/minimumInputLength',
   './data/maximumInputLength',
 
@@ -3229,7 +3317,7 @@ define('select2/defaults',[
 
              Utils, Translation, DIACRITICS,
 
-             SelectData, ArrayData, AjaxData, Tags,
+             SelectData, ArrayData, AjaxData, Tags, Tokenizer,
              MinimumInputLength, MaximumInputLength,
 
              Dropdown, DropdownSearch, HidePlaceholder, InfiniteScroll,
@@ -3269,6 +3357,13 @@ define('select2/defaults',[
       if (options.tags != null) {
         options.dataAdapter = Utils.Decorate(options.dataAdapter, Tags);
       }
+
+      if (options.tokenSeparators != null || options.tokenizer != null) {
+        options.dataAdapter = Utils.Decorate(
+          options.dataAdapter,
+          Tokenizer
+        );
+      }
     }
 
     if (options.resultsAdapter == null) {

dist/js/select2.amd.js

@@ -2646,6 +2646,93 @@ define('select2/data/tags',[
   return Tags;
 });
 
+define('select2/data/tokenizer',[
+
+], function () {
+  function Tokenizer (decorated, $element, options) {
+    var tokenizer = options.get('tokenizer');
+
+    if (tokenizer !== undefined) {
+      this.tokenizer = tokenizer;
+    }
+
+    decorated.call(this, $element, options);
+  }
+
+  Tokenizer.prototype.bind = function (decorated, container, $container) {
+    decorated.call(this, container, $container);
+
+    this.$search =  container.dropdown.$search || container.selection.$search ||
+      $container.find('.select2-search__field');
+  };
+
+  Tokenizer.prototype.query = function (decorated, params, callback) {
+    var self = this;
+
+    function select (data) {
+      self.select(data);
+    }
+
+    params.term = params.term || '';
+
+    var tokenData = this.tokenizer(params, this.options, select);
+
+    if (tokenData.term !== params.term) {
+      // Replace the search term if we have the search box
+      if (this.$search.length) {
+        this.$search.val(tokenData.term);
+        this.$search.focus();
+      }
+
+      params.term = tokenData.term;
+    }
+
+    decorated.call(this, params, callback);
+  };
+
+  Tokenizer.prototype.tokenizer = function (_, params, options, callback) {
+    var separators = options.get('tokenSeparators') || [];
+    var term = params.term;
+    var i = 0;
+
+    var createTag = this.createTag || function (params) {
+      return {
+        id: params.term,
+        text: params.term
+      };
+    };
+
+    while (i < term.length) {
+      var termChar = term[i];
+
+      if (separators.indexOf(termChar) === -1) {
+        i++;
+
+        continue;
+      }
+
+      var part = term.substr(0, i);
+      var partParams = $.extend({}, params, {
+        term: part
+      });
+
+      var data = createTag(partParams);
+
+      callback(data);
+
+      // Reset the term to not include the tokenized portion
+      term = term.substr(i + 1) || '';
+      i = 0;
+    }
+
+    return {
+      term: term
+    };
+  };
+
+  return Tokenizer;
+});
+
 define('select2/data/minimumInputLength',[
 
 ], function () {
@@ -3211,6 +3298,7 @@ define('select2/defaults',[
   './data/array',
   './data/ajax',
   './data/tags',
+  './data/tokenizer',
   './data/minimumInputLength',
   './data/maximumInputLength',
 
@@ -3229,7 +3317,7 @@ define('select2/defaults',[
 
              Utils, Translation, DIACRITICS,
 
-             SelectData, ArrayData, AjaxData, Tags,
+             SelectData, ArrayData, AjaxData, Tags, Tokenizer,
              MinimumInputLength, MaximumInputLength,
 
              Dropdown, DropdownSearch, HidePlaceholder, InfiniteScroll,
@@ -3269,6 +3357,13 @@ define('select2/defaults',[
       if (options.tags != null) {
         options.dataAdapter = Utils.Decorate(options.dataAdapter, Tags);
       }
+
+      if (options.tokenSeparators != null || options.tokenizer != null) {
+        options.dataAdapter = Utils.Decorate(
+          options.dataAdapter,
+          Tokenizer
+        );
+      }
     }
 
     if (options.resultsAdapter == null) {

dist/js/select2.full.js

@@ -12181,6 +12181,93 @@ define('select2/data/tags',[
   return Tags;
 });
 
+define('select2/data/tokenizer',[
+
+], function () {
+  function Tokenizer (decorated, $element, options) {
+    var tokenizer = options.get('tokenizer');
+
+    if (tokenizer !== undefined) {
+      this.tokenizer = tokenizer;
+    }
+
+    decorated.call(this, $element, options);
+  }
+
+  Tokenizer.prototype.bind = function (decorated, container, $container) {
+    decorated.call(this, container, $container);
+
+    this.$search =  container.dropdown.$search || container.selection.$search ||
+      $container.find('.select2-search__field');
+  };
+
+  Tokenizer.prototype.query = function (decorated, params, callback) {
+    var self = this;
+
+    function select (data) {
+      self.select(data);
+    }
+
+    params.term = params.term || '';
+
+    var tokenData = this.tokenizer(params, this.options, select);
+
+    if (tokenData.term !== params.term) {
+      // Replace the search term if we have the search box
+      if (this.$search.length) {
+        this.$search.val(tokenData.term);
+        this.$search.focus();
+      }
+
+      params.term = tokenData.term;
+    }
+
+    decorated.call(this, params, callback);
+  };
+
+  Tokenizer.prototype.tokenizer = function (_, params, options, callback) {
+    var separators = options.get('tokenSeparators') || [];
+    var term = params.term;
+    var i = 0;
+
+    var createTag = this.createTag || function (params) {
+      return {
+        id: params.term,
+        text: params.term
+      };
+    };
+
+    while (i < term.length) {
+      var termChar = term[i];
+
+      if (separators.indexOf(termChar) === -1) {
+        i++;
+
+        continue;
+      }
+
+      var part = term.substr(0, i);
+      var partParams = $.extend({}, params, {
+        term: part
+      });
+
+      var data = createTag(partParams);
+
+      callback(data);
+
+      // Reset the term to not include the tokenized portion
+      term = term.substr(i + 1) || '';
+      i = 0;
+    }
+
+    return {
+      term: term
+    };
+  };
+
+  return Tokenizer;
+});
+
 define('select2/data/minimumInputLength',[
 
 ], function () {
@@ -12746,6 +12833,7 @@ define('select2/defaults',[
   './data/array',
   './data/ajax',
   './data/tags',
+  './data/tokenizer',
   './data/minimumInputLength',
   './data/maximumInputLength',
 
@@ -12764,7 +12852,7 @@ define('select2/defaults',[
 
              Utils, Translation, DIACRITICS,
 
-             SelectData, ArrayData, AjaxData, Tags,
+             SelectData, ArrayData, AjaxData, Tags, Tokenizer,
              MinimumInputLength, MaximumInputLength,
 
              Dropdown, DropdownSearch, HidePlaceholder, InfiniteScroll,
@@ -12804,6 +12892,13 @@ define('select2/defaults',[
       if (options.tags != null) {
         options.dataAdapter = Utils.Decorate(options.dataAdapter, Tags);
       }
+
+      if (options.tokenSeparators != null || options.tokenizer != null) {
+        options.dataAdapter = Utils.Decorate(
+          options.dataAdapter,
+          Tokenizer
+        );
+      }
     }
 
     if (options.resultsAdapter == null) {

dist/js/select2.js

@@ -3074,6 +3074,93 @@ define('select2/data/tags',[
   return Tags;
 });
 
+define('select2/data/tokenizer',[
+
+], function () {
+  function Tokenizer (decorated, $element, options) {
+    var tokenizer = options.get('tokenizer');
+
+    if (tokenizer !== undefined) {
+      this.tokenizer = tokenizer;
+    }
+
+    decorated.call(this, $element, options);
+  }
+
+  Tokenizer.prototype.bind = function (decorated, container, $container) {
+    decorated.call(this, container, $container);
+
+    this.$search =  container.dropdown.$search || container.selection.$search ||
+      $container.find('.select2-search__field');
+  };
+
+  Tokenizer.prototype.query = function (decorated, params, callback) {
+    var self = this;
+
+    function select (data) {
+      self.select(data);
+    }
+
+    params.term = params.term || '';
+
+    var tokenData = this.tokenizer(params, this.options, select);
+
+    if (tokenData.term !== params.term) {
+      // Replace the search term if we have the search box
+      if (this.$search.length) {
+        this.$search.val(tokenData.term);
+        this.$search.focus();
+      }
+
+      params.term = tokenData.term;
+    }
+
+    decorated.call(this, params, callback);
+  };
+
+  Tokenizer.prototype.tokenizer = function (_, params, options, callback) {
+    var separators = options.get('tokenSeparators') || [];
+    var term = params.term;
+    var i = 0;
+
+    var createTag = this.createTag || function (params) {
+      return {
+        id: params.term,
+        text: params.term
+      };
+    };
+
+    while (i < term.length) {
+      var termChar = term[i];
+
+      if (separators.indexOf(termChar) === -1) {
+        i++;
+
+        continue;
+      }
+
+      var part = term.substr(0, i);
+      var partParams = $.extend({}, params, {
+        term: part
+      });
+
+      var data = createTag(partParams);
+
+      callback(data);
+
+      // Reset the term to not include the tokenized portion
+      term = term.substr(i + 1) || '';
+      i = 0;
+    }
+
+    return {
+      term: term
+    };
+  };
+
+  return Tokenizer;
+});
+
 define('select2/data/minimumInputLength',[
 
 ], function () {
@@ -3639,6 +3726,7 @@ define('select2/defaults',[
   './data/array',
   './data/ajax',
   './data/tags',
+  './data/tokenizer',
   './data/minimumInputLength',
   './data/maximumInputLength',
 
@@ -3657,7 +3745,7 @@ define('select2/defaults',[
 
              Utils, Translation, DIACRITICS,
 
-             SelectData, ArrayData, AjaxData, Tags,
+             SelectData, ArrayData, AjaxData, Tags, Tokenizer,
              MinimumInputLength, MaximumInputLength,
 
              Dropdown, DropdownSearch, HidePlaceholder, InfiniteScroll,
@@ -3697,6 +3785,13 @@ define('select2/defaults',[
       if (options.tags != null) {
         options.dataAdapter = Utils.Decorate(options.dataAdapter, Tags);
       }
+
+      if (options.tokenSeparators != null || options.tokenizer != null) {
+        options.dataAdapter = Utils.Decorate(
+          options.dataAdapter,
+          Tokenizer
+        );
+      }
     }
 
     if (options.resultsAdapter == null) {

docs/examples.html

@@ -429,6 +429,45 @@ $(".js-example-tags").select2({
     </div>
   </section>
 
+  <section id="tokenizer" class="row">
+    <div class="col-md-4">
+      <h1>Automatic tokenization</h1>
+
+      <p>
+        Select2 supports ability to add choices automatically as the user is
+        typing into the search field. Try typing in the search field below and
+        entering a space or a comma.
+      </p>
+
+      <p>
+        <select class="js-example-tokenizer form-control" multiple="multiple">
+          <option>red</option>
+          <option>blue</option>
+          <option>green</option>
+        </select>
+      </p>
+
+      <p>
+        The separators that should be used when tokenizing can be specified
+        using the <a href="options.html#tokenSeparators">tokenSeparators</a>
+        options.
+      </p>
+
+    </div>
+    <div class="col-md-8">
+      <h2>Example code</h2>
+
+      <pre data-fill-from=".js-code-tokenizer"></pre>
+
+<script type="text/x-example-code" class="js-code-tokenizer">
+$(".js-example-tokenizer").select2({
+  tags: true,
+  tokenSeparators: [',', ' ']
+})
+</script>
+    </div>
+  </section>
+
   <section id="matcher" class="row">
     <div class="col-md-4">
       <h1>Custom matcher</h1>
@@ -801,6 +840,11 @@ $.fn.select2.amd.require(
     tags: ['red', 'blue', 'green']
   });
 
+  $(".js-example-tokenizer").select2({
+    tags: true,
+    tokenSeparators: [',', ' ']
+  });
+
   function matchStart (term, text) {
     if (text.toUpperCase().indexOf(term.toUpperCase()) == 0) {
       return true;

src/js/select2/data/tokenizer.js

@@ -0,0 +1,86 @@
+define([
+
+], function () {
+  function Tokenizer (decorated, $element, options) {
+    var tokenizer = options.get('tokenizer');
+
+    if (tokenizer !== undefined) {
+      this.tokenizer = tokenizer;
+    }
+
+    decorated.call(this, $element, options);
+  }
+
+  Tokenizer.prototype.bind = function (decorated, container, $container) {
+    decorated.call(this, container, $container);
+
+    this.$search =  container.dropdown.$search || container.selection.$search ||
+      $container.find('.select2-search__field');
+  };
+
+  Tokenizer.prototype.query = function (decorated, params, callback) {
+    var self = this;
+
+    function select (data) {
+      self.select(data);
+    }
+
+    params.term = params.term || '';
+
+    var tokenData = this.tokenizer(params, this.options, select);
+
+    if (tokenData.term !== params.term) {
+      // Replace the search term if we have the search box
+      if (this.$search.length) {
+        this.$search.val(tokenData.term);
+        this.$search.focus();
+      }
+
+      params.term = tokenData.term;
+    }
+
+    decorated.call(this, params, callback);
+  };
+
+  Tokenizer.prototype.tokenizer = function (_, params, options, callback) {
+    var separators = options.get('tokenSeparators') || [];
+    var term = params.term;
+    var i = 0;
+
+    var createTag = this.createTag || function (params) {
+      return {
+        id: params.term,
+        text: params.term
+      };
+    };
+
+    while (i < term.length) {
+      var termChar = term[i];
+
+      if (separators.indexOf(termChar) === -1) {
+        i++;
+
+        continue;
+      }
+
+      var part = term.substr(0, i);
+      var partParams = $.extend({}, params, {
+        term: part
+      });
+
+      var data = createTag(partParams);
+
+      callback(data);
+
+      // Reset the term to not include the tokenized portion
+      term = term.substr(i + 1) || '';
+      i = 0;
+    }
+
+    return {
+      term: term
+    };
+  };
+
+  return Tokenizer;
+});

src/js/select2/defaults.js

@@ -16,6 +16,7 @@ define([
   './data/array',
   './data/ajax',
   './data/tags',
+  './data/tokenizer',
   './data/minimumInputLength',
   './data/maximumInputLength',
 
@@ -34,7 +35,7 @@ define([
 
              Utils, Translation, DIACRITICS,
 
-             SelectData, ArrayData, AjaxData, Tags,
+             SelectData, ArrayData, AjaxData, Tags, Tokenizer,
              MinimumInputLength, MaximumInputLength,
 
              Dropdown, DropdownSearch, HidePlaceholder, InfiniteScroll,
@@ -74,6 +75,13 @@ define([
       if (options.tags != null) {
         options.dataAdapter = Utils.Decorate(options.dataAdapter, Tags);
       }
+
+      if (options.tokenSeparators != null || options.tokenizer != null) {
+        options.dataAdapter = Utils.Decorate(
+          options.dataAdapter,
+          Tokenizer
+        );
+      }
     }
 
     if (options.resultsAdapter == null) {