elasticlunr = function (config) {
var idx = new elasticlunr.Index;
idx.pipeline.add(
elasticlunr.trimmer,
elasticlunr.stopWordFilter,
elasticlunr.stemmer
);
if (config) config.call(idx, idx);
return idx;
}n/a
Configuration = function (config, fields) {
var config = config || '';
if (fields == undefined || fields == null) {
throw new Error('fields should not be null');
}
this.config = {};
var userConfig;
try {
userConfig = JSON.parse(config);
this.buildUserConfig(userConfig, fields);
} catch (error) {
elasticlunr.utils.warn('user configuration parse failed, will use default configuration');
this.buildDefaultConfig(fields);
}
}...
if (!query) return [];
var configStr = null;
if (userConfig != null) {
configStr = JSON.stringify(userConfig);
}
var config = new elasticlunr.Configuration(configStr, this.getFields()).get();
var queryTokens = this.pipeline.run(elasticlunr.tokenizer(query));
var queryResults = {};
for (var field in config) {
var fieldSearchResults = this.fieldSearch(queryTokens, field, config);
...DocumentStore = function (save) {
if (save === null || save === undefined) {
this._save = true;
} else {
this._save = save;
}
this.docs = {};
this.docInfo = {};
this.length = 0;
}...
* Defaultly save all the original JSON documents.
*
* @param {Boolean} save Whether to save the original JSON documents.
* @return {elasticlunr.Index}
* @memberOf Index
*/
elasticlunr.Index.prototype.saveDocument = function (save) {
this.documentStore = new elasticlunr.DocumentStore(save);
return this;
};
/**
* Add a JSON format document to the index.
*
* This is the way new documents enter the index, this function will run the
...EventEmitter = function () {
this.events = {};
}n/a
Index = function () {
this._fields = [];
this._ref = 'id';
this.pipeline = new elasticlunr.Pipeline;
this.documentStore = new elasticlunr.DocumentStore;
this.index = {};
this.eventEmitter = new elasticlunr.EventEmitter;
this._idfCache = {};
this.on('add', 'remove', 'update', (function () {
this._idfCache = {};
}).bind(this));
}n/a
InvertedIndex = function () {
this.root = { docs: {}, df: 0 };
}n/a
Pipeline = function () {
this._queue = [];
}n/a
SortedSet = function () {
this.length = 0
this.elements = []
}n/a
addStopWords = function (words) {
if (words == null || Array.isArray(words) === false) return;
words.forEach(function (word) {
elasticlunr.stopWordFilter.stopWords[word] = true;
}, this);
}...
```
### 6.2 Add customized stop words
User could add a list of customized stop words.
```javascript
var customized_stop_words = ['an', 'hello', 'xyzabc'];
elasticlunr.addStopWords(customized_stop_words);
```
## 7. Use elasticlunr in Node.js
Elasticlunr support Node.js, you could use elastilunr in node.js as a node-module.
Install elasticlunr by:
...clearStopWords = function () {
elasticlunr.stopWordFilter.stopWords = {};
}...
Defaultly elasticlunr.js contains **120** stop words, user could decide not use these default stop words or add customized stop
words.
### 6.1 Remove default stop words
You could remove default stop words simply as:
```javascript
elasticlunr.clearStopWords();
```
### 6.2 Add customized stop words
User could add a list of customized stop words.
```javascript
var customized_stop_words = ['an', 'hello', 'xyzabc'];
...resetStopWords = function () {
elasticlunr.stopWordFilter.stopWords = elasticlunr.defaultStopWords;
}n/a
function porterStemmer(w) {
var stem,
suffix,
firstch,
re,
re2,
re3,
re4;
if (w.length < 3) { return w; }
firstch = w.substr(0,1);
if (firstch == "y") {
w = firstch.toUpperCase() + w.substr(1);
}
// Step 1a
re = re_1a
re2 = re2_1a;
if (re.test(w)) { w = w.replace(re,"$1$2"); }
else if (re2.test(w)) { w = w.replace(re2,"$1$2"); }
// Step 1b
re = re_1b;
re2 = re2_1b;
if (re.test(w)) {
var fp = re.exec(w);
re = re_mgr0;
if (re.test(fp[1])) {
re = re_1b_2;
w = w.replace(re,"");
}
} else if (re2.test(w)) {
var fp = re2.exec(w);
stem = fp[1];
re2 = re_s_v;
if (re2.test(stem)) {
w = stem;
re2 = re2_1b_2;
re3 = re3_1b_2;
re4 = re4_1b_2;
if (re2.test(w)) { w = w + "e"; }
else if (re3.test(w)) { re = re_1b_2; w = w.replace(re,""); }
else if (re4.test(w)) { w = w + "e"; }
}
}
// Step 1c - replace suffix y or Y by i if preceded by a non-vowel which is not the first letter of the word (so cry -> cri, by
-> by, say -> say)
re = re_1c;
if (re.test(w)) {
var fp = re.exec(w);
stem = fp[1];
w = stem + "i";
}
// Step 2
re = re_2;
if (re.test(w)) {
var fp = re.exec(w);
stem = fp[1];
suffix = fp[2];
re = re_mgr0;
if (re.test(stem)) {
w = stem + step2list[suffix];
}
}
// Step 3
re = re_3;
if (re.test(w)) {
var fp = re.exec(w);
stem = fp[1];
suffix = fp[2];
re = re_mgr0;
if (re.test(stem)) {
w = stem + step3list[suffix];
}
}
// Step 4
re = re_4;
re2 = re2_4;
if (re.test(w)) {
var fp = re.exec(w);
stem = fp[1];
re = re_mgr1;
if (re.test(stem)) {
w = stem;
}
} else if (re2.test(w)) {
var fp = re2.exec(w);
stem = fp[1] + fp[2];
re2 = re_mgr1;
if (re2.test(stem)) {
w = stem;
}
}
// Step 5
re = re_5;
if (re.test(w)) {
var fp = re.exec(w);
stem = fp[1];
re = re_mgr1;
re2 = re_meq1;
re3 = re3_5;
if (re.test(stem) || (re2.test(stem) && !(re3.test(stem)))) {
w = stem;
}
}
re = re_5_1;
re2 = re_mgr1;
if (re.test(w) && re2.test(w)) {
re = re_1b_2;
w = w.replace(re,"");
}
// and turn initial Y back to y
if (firstch == "y") {
w = firstch.toLowerCase() + w.substr(1);
}
return w;
}n/a
stopWordFilter = function (token) {
if (token && elasticlunr.stopWordFilter.stopWords[token] !== true) {
return token;
}
}n/a
tokenizer = function (str) {
if (!arguments.length || str === null || str === undefined) return [];
if (Array.isArray(str)) {
var arr = str.filter(function(token) {
if (token === null || token === undefined) {
return false;
}
return true;
});
arr = arr.map(function (t) {
return elasticlunr.utils.toString(t).toLowerCase();
});
var out = [];
arr.forEach(function(item) {
var tokens = item.split(elasticlunr.tokenizer.seperator);
out = out.concat(tokens);
}, this);
return out;
}
return str.toString().trim().toLowerCase().split(elasticlunr.tokenizer.seperator);
}...
if (!doc) return;
var emitEvent = emitEvent === undefined ? true : emitEvent;
var docRef = doc[this._ref];
this.documentStore.addDoc(docRef, doc);
this._fields.forEach(function (field) {
var fieldTokens = this.pipeline.run(elasticlunr.tokenizer(doc[field]));
this.documentStore.addFieldLength(docRef, field, fieldTokens.length);
var tokenCount = {};
fieldTokens.forEach(function (token) {
if (token in tokenCount) tokenCount[token] += 1;
else tokenCount[token] = 1;
}, this);
...trimmer = function (token) {
if (token === null || token === undefined) {
throw new Error('token should not be undefined');
}
return token
.replace(/^\W+/, '')
.replace(/\W+$/, '');
}n/a
Configuration = function (config, fields) {
var config = config || '';
if (fields == undefined || fields == null) {
throw new Error('fields should not be null');
}
this.config = {};
var userConfig;
try {
userConfig = JSON.parse(config);
this.buildUserConfig(userConfig, fields);
} catch (error) {
elasticlunr.utils.warn('user configuration parse failed, will use default configuration');
this.buildDefaultConfig(fields);
}
}...
if (!query) return [];
var configStr = null;
if (userConfig != null) {
configStr = JSON.stringify(userConfig);
}
var config = new elasticlunr.Configuration(configStr, this.getFields()).get();
var queryTokens = this.pipeline.run(elasticlunr.tokenizer(query));
var queryResults = {};
for (var field in config) {
var fieldSearchResults = this.fieldSearch(queryTokens, field, config);
...addAllFields2UserConfig = function (bool, expand, fields) {
fields.forEach(function (field) {
this.config[field] = {
boost: 1,
bool: bool,
expand: expand
};
}, this);
}...
expand: field_expand
};
} else {
elasticlunr.utils.warn('field name in user configuration not found in index instance fields');
}
}
} else {
this.addAllFields2UserConfig(global_bool, global_expand, fields);
}
};
/**
* Add all fields to user search configuration.
*
* @param {String} bool Boolean model
...buildDefaultConfig = function (fields) {
this.reset();
fields.forEach(function (field) {
this.config[field] = {
boost: 1,
bool: "OR",
expand: false
};
}, this);
}...
var userConfig;
try {
userConfig = JSON.parse(config);
this.buildUserConfig(userConfig, fields);
} catch (error) {
elasticlunr.utils.warn('user configuration parse failed, will use default configuration');
this.buildDefaultConfig(fields);
}
};
/**
* Build default search configuration.
*
* @param {Array} fields fields of index instance
...buildUserConfig = function (config, fields) {
var global_bool = "OR";
var global_expand = false;
this.reset();
if ('bool' in config) {
global_bool = config['bool'] || global_bool;
}
if ('expand' in config) {
global_expand = config['expand'] || global_expand;
}
if ('fields' in config) {
for (var field in config['fields']) {
if (fields.indexOf(field) > -1) {
var field_config = config['fields'][field];
var field_expand = global_expand;
if (field_config.expand != undefined) {
field_expand = field_config.expand;
}
this.config[field] = {
boost: (field_config.boost || field_config.boost === 0) ? field_config.boost : 1,
bool: field_config.bool || global_bool,
expand: field_expand
};
} else {
elasticlunr.utils.warn('field name in user configuration not found in index instance fields');
}
}
} else {
this.addAllFields2UserConfig(global_bool, global_expand, fields);
}
}...
}
this.config = {};
var userConfig;
try {
userConfig = JSON.parse(config);
this.buildUserConfig(userConfig, fields);
} catch (error) {
elasticlunr.utils.warn('user configuration parse failed, will use default configuration');
this.buildDefaultConfig(fields);
}
};
/**
...get = function () {
return this.config;
}...
if (!query) return [];
var configStr = null;
if (userConfig != null) {
configStr = JSON.stringify(userConfig);
}
var config = new elasticlunr.Configuration(configStr, this.getFields()).get();
var queryTokens = this.pipeline.run(elasticlunr.tokenizer(query));
var queryResults = {};
for (var field in config) {
var fieldSearchResults = this.fieldSearch(queryTokens, field, config);
...reset = function () {
this.config = {};
}...
/**
* Build default search configuration.
*
* @param {Array} fields fields of index instance
*/
elasticlunr.Configuration.prototype.buildDefaultConfig = function (fields) {
this.reset();
fields.forEach(function (field) {
this.config[field] = {
boost: 1,
bool: "OR",
expand: false
};
}, this);
...DocumentStore = function (save) {
if (save === null || save === undefined) {
this._save = true;
} else {
this._save = save;
}
this.docs = {};
this.docInfo = {};
this.length = 0;
}...
* Defaultly save all the original JSON documents.
*
* @param {Boolean} save Whether to save the original JSON documents.
* @return {elasticlunr.Index}
* @memberOf Index
*/
elasticlunr.Index.prototype.saveDocument = function (save) {
this.documentStore = new elasticlunr.DocumentStore(save);
return this;
};
/**
* Add a JSON format document to the index.
*
* This is the way new documents enter the index, this function will run the
...load = function (serialisedData) {
var store = new this;
store.length = serialisedData.length;
store.docs = serialisedData.docs;
store.docInfo = serialisedData.docInfo;
store._save = serialisedData.save;
return store;
}...
+ elasticlunr.version + ' importing ' + serialisedData.version);
}
var idx = new this;
idx._fields = serialisedData.fields;
idx._ref = serialisedData.ref;
idx.documentStore = elasticlunr.DocumentStore.load(serialisedData.documentStore);
idx.pipeline = elasticlunr.Pipeline.load(serialisedData.pipeline);
idx.index = {};
for (var field in serialisedData.index) {
idx.index[field] = elasticlunr.InvertedIndex.load(serialisedData.index[field]);
}
return idx;
...addDoc = function (docRef, doc) {
if (!this.hasDoc(docRef)) this.length++;
if (this._save === true) {
this.docs[docRef] = clone(doc);
} else {
this.docs[docRef] = null;
}
}...
var doc2 = {
"id": 2,
"title": "Oracle released its profit report of 2015",
"body": "As expected, Oracle released its profit report of 2015, during the good sales of database and hardware
, Oracle's profit of 2015 reached 12.5 Billion."
}
index.addDoc(doc1);
index.addDoc(doc2);
```
Then searching is as simple:
```javascript
index.search("Oracle database profit");
...addFieldLength = function (docRef, fieldName, length) {
if (docRef === null || docRef === undefined) return;
if (this.hasDoc(docRef) == false) return;
if (!this.docInfo[docRef]) this.docInfo[docRef] = {};
this.docInfo[docRef][fieldName] = length;
}...
var emitEvent = emitEvent === undefined ? true : emitEvent;
var docRef = doc[this._ref];
this.documentStore.addDoc(docRef, doc);
this._fields.forEach(function (field) {
var fieldTokens = this.pipeline.run(elasticlunr.tokenizer(doc[field]));
this.documentStore.addFieldLength(docRef, field, fieldTokens.length);
var tokenCount = {};
fieldTokens.forEach(function (token) {
if (token in tokenCount) tokenCount[token] += 1;
else tokenCount[token] = 1;
}, this);
...getDoc = function (docRef) {
if (this.hasDoc(docRef) === false) return null;
return this.docs[docRef];
}...
elasticlunr.Index.prototype.removeDocByRef = function (docRef, emitEvent) {
if (!docRef) return;
if (this.documentStore.isDocStored() === false) {
return;
}
if (!this.documentStore.hasDoc(docRef)) return;
var doc = this.documentStore.getDoc(docRef);
this.removeDoc(doc, false);
};
/**
* Removes a document from the index.
* This remove operation could work even the original doc is not store in the DocumentStore.
*
...getFieldLength = function (docRef, fieldName) {
if (docRef === null || docRef === undefined) return 0;
if (!(docRef in this.docs)) return 0;
if (!(fieldName in this.docInfo[docRef])) return 0;
return this.docInfo[docRef][fieldName];
}...
// coordNorm
if (key == token) {
this.fieldSearchStats(docTokens, key, docs);
}
for (var docRef in docs) {
var tf = this.index[fieldName].getTermFrequency(key, docRef);
var fieldLength = this.documentStore.getFieldLength(docRef, fieldName);
var fieldLengthNorm = 1;
if (fieldLength != 0) {
fieldLengthNorm = 1 / Math.sqrt(fieldLength);
}
var penality = 1;
if (key != token) {
...hasDoc = function (docRef) {
return docRef in this.docs;
}...
*/
elasticlunr.Index.prototype.removeDocByRef = function (docRef, emitEvent) {
if (!docRef) return;
if (this.documentStore.isDocStored() === false) {
return;
}
if (!this.documentStore.hasDoc(docRef)) return;
var doc = this.documentStore.getDoc(docRef);
this.removeDoc(doc, false);
};
/**
* Removes a document from the index.
* This remove operation could work even the original doc is not store in the DocumentStore.
...isDocStored = function () {
return this._save;
}...
*
* @param {String|Integer} docRef The document ref to remove from the index.
* @param {Boolean} emitEvent Whether to emit remove events, defaults to true
* @memberOf Index
*/
elasticlunr.Index.prototype.removeDocByRef = function (docRef, emitEvent) {
if (!docRef) return;
if (this.documentStore.isDocStored() === false) {
return;
}
if (!this.documentStore.hasDoc(docRef)) return;
var doc = this.documentStore.getDoc(docRef);
this.removeDoc(doc, false);
};
...removeDoc = function (docRef) {
if (!this.hasDoc(docRef)) return;
delete this.docs[docRef];
delete this.docInfo[docRef];
this.length--;
}...
index.addDoc(doc2);
```
If your JSON document contains field that not configured in index, then that field will not be indexed, which means that field is
not searchable.
## 3. Remove document from index
Elasticlunr.js support remove a document from index, just provide JSON document to <code>elasticlunr.Index.prototype.<
span class="apidocCodeKeywordSpan">removeDoc()</code> function.
For example:
```javascript
var doc = {
"id": 1,
"title": "Oracle released its latest database Oracle 12g",
"body": "Yestaday Oracle has released its new database Oracle 12g, this would make more money for this company and
lead to a nice profit report of annual year."
...toJSON = function () {
return {
docs: this.docs,
docInfo: this.docInfo,
length: this.length,
save: this._save
};
}...
*
* @return {Object}
* @memberOf Index
*/
elasticlunr.Index.prototype.toJSON = function () {
var indexJson = {};
this._fields.forEach(function (field) {
indexJson[field] = this.index[field].toJSON();
}, this);
return {
version: elasticlunr.version,
fields: this._fields,
ref: this._ref,
documentStore: this.documentStore.toJSON(),
...updateFieldLength = function (docRef, fieldName, length) {
if (docRef === null || docRef === undefined) return;
if (this.hasDoc(docRef) == false) return;
this.addFieldLength(docRef, fieldName, length);
}n/a
EventEmitter = function () {
this.events = {};
}n/a
addListener = function () {
var args = Array.prototype.slice.call(arguments),
fn = args.pop(),
names = args;
if (typeof fn !== "function") throw new TypeError ("last argument must be a function");
names.forEach(function (name) {
if (!this.hasHandler(name)) this.events[name] = [];
this.events[name].push(fn);
}, this);
}n/a
emit = function (name) {
if (!this.hasHandler(name)) return;
var args = Array.prototype.slice.call(arguments, 1);
this.events[name].forEach(function (fn) {
fn.apply(undefined, args);
}, this);
}...
for (var token in tokenCount) {
var termFrequency = tokenCount[token];
termFrequency = Math.sqrt(termFrequency);
this.index[field].addToken(token, { ref: docRef, tf: termFrequency });
}
}, this);
if (emitEvent) this.eventEmitter.emit('add', doc, this);
};
/**
* Removes a document from the index by doc ref.
*
* To make sure documents no longer show up in search results they can be
* removed from the index using this method.
...hasHandler = function (name) {
return name in this.events;
}...
var args = Array.prototype.slice.call(arguments),
fn = args.pop(),
names = args;
if (typeof fn !== "function") throw new TypeError ("last argument must be a function");
names.forEach(function (name) {
if (!this.hasHandler(name)) this.events[name] = [];
this.events[name].push(fn);
}, this);
};
/**
* Removes a handler function from a specific event.
*
...removeListener = function (name, fn) {
if (!this.hasHandler(name)) return;
var fnIndex = this.events[name].indexOf(fn);
if (fnIndex === -1) return;
this.events[name].splice(fnIndex, 1);
if (this.events[name].length == 0) delete this.events[name];
}...
* Removes a handler from an event being emitted by the index.
*
* @param {String} eventName The name of events to remove the function from.
* @param {Function} fn The serialised set to load.
* @memberOf Index
*/
elasticlunr.Index.prototype.off = function (name, fn) {
return this.eventEmitter.removeListener(name, fn);
};
/**
* Loads a previously serialised index.
*
* Issues a warning if the index being imported was serialised
* by a different version of elasticlunr.
...Index = function () {
this._fields = [];
this._ref = 'id';
this.pipeline = new elasticlunr.Pipeline;
this.documentStore = new elasticlunr.DocumentStore;
this.index = {};
this.eventEmitter = new elasticlunr.EventEmitter;
this._idfCache = {};
this.on('add', 'remove', 'update', (function () {
this._idfCache = {};
}).bind(this));
}n/a
load = function (serialisedData) {
if (serialisedData.version !== elasticlunr.version) {
elasticlunr.utils.warn('version mismatch: current '
+ elasticlunr.version + ' importing ' + serialisedData.version);
}
var idx = new this;
idx._fields = serialisedData.fields;
idx._ref = serialisedData.ref;
idx.documentStore = elasticlunr.DocumentStore.load(serialisedData.documentStore);
idx.pipeline = elasticlunr.Pipeline.load(serialisedData.pipeline);
idx.index = {};
for (var field in serialisedData.index) {
idx.index[field] = elasticlunr.InvertedIndex.load(serialisedData.index[field]);
}
return idx;
}...
+ elasticlunr.version + ' importing ' + serialisedData.version);
}
var idx = new this;
idx._fields = serialisedData.fields;
idx._ref = serialisedData.ref;
idx.documentStore = elasticlunr.DocumentStore.load(serialisedData.documentStore);
idx.pipeline = elasticlunr.Pipeline.load(serialisedData.pipeline);
idx.index = {};
for (var field in serialisedData.index) {
idx.index[field] = elasticlunr.InvertedIndex.load(serialisedData.index[field]);
}
return idx;
...addDoc = function (doc, emitEvent) {
if (!doc) return;
var emitEvent = emitEvent === undefined ? true : emitEvent;
var docRef = doc[this._ref];
this.documentStore.addDoc(docRef, doc);
this._fields.forEach(function (field) {
var fieldTokens = this.pipeline.run(elasticlunr.tokenizer(doc[field]));
this.documentStore.addFieldLength(docRef, field, fieldTokens.length);
var tokenCount = {};
fieldTokens.forEach(function (token) {
if (token in tokenCount) tokenCount[token] += 1;
else tokenCount[token] = 1;
}, this);
for (var token in tokenCount) {
var termFrequency = tokenCount[token];
termFrequency = Math.sqrt(termFrequency);
this.index[field].addToken(token, { ref: docRef, tf: termFrequency });
}
}, this);
if (emitEvent) this.eventEmitter.emit('add', doc, this);
}...
var doc2 = {
"id": 2,
"title": "Oracle released its profit report of 2015",
"body": "As expected, Oracle released its profit report of 2015, during the good sales of database and hardware
, Oracle's profit of 2015 reached 12.5 Billion."
}
index.addDoc(doc1);
index.addDoc(doc2);
```
Then searching is as simple:
```javascript
index.search("Oracle database profit");
...addField = function (fieldName) {
this._fields.push(fieldName);
this.index[fieldName] = new elasticlunr.InvertedIndex;
return this;
}...
## Example
A very simple search index can be created using the following scripts:
```javascript
var index = elasticlunr(function () {
this.addField('title');
this.addField('body');
this.setRef('id');
});
```
Adding documents to the index is as simple as:
...coordNorm = function (scores, docTokens, n) {
for (var doc in scores) {
if (!(doc in docTokens)) continue;
var tokens = docTokens[doc].length;
scores[doc] = scores[doc] * tokens / n;
}
return scores;
}...
}
}
}, this);
scores = this.mergeScores(scores, queryTokenScores, booleanType);
}, this);
scores = this.coordNorm(scores, docTokens, queryTokens.length);
return scores;
};
/**
* Merge the scores from one set of tokens into an accumulated score table.
* Exact operation depends on the op parameter. If op is 'AND', then only the
* intersection of the two score lists is retained. Otherwise, the union of
...fieldSearch = function (queryTokens, fieldName, config) {
var booleanType = config[fieldName].bool;
var expand = config[fieldName].expand;
var boost = config[fieldName].boost;
var scores = null;
var docTokens = {};
// Do nothing if the boost is 0
if (boost === 0) {
return;
}
queryTokens.forEach(function (token) {
var tokens = [token];
if (expand == true) {
tokens = this.index[fieldName].expandToken(token);
}
// Consider every query token in turn. If expanded, each query token
// corresponds to a set of tokens, which is all tokens in the
// index matching the pattern queryToken* .
// For the set of tokens corresponding to a query token, find and score
// all matching documents. Store those scores in queryTokenScores,
// keyed by docRef.
// Then, depending on the value of booleanType, combine the scores
// for this query token with previous scores. If booleanType is OR,
// then merge the scores by summing into the accumulated total, adding
// new document scores are required (effectively a union operator).
// If booleanType is AND, accumulate scores only if the document
// has previously been scored by another query token (an intersection
// operation0.
// Furthermore, since when booleanType is AND, additional
// query tokens can't add new documents to the result set, use the
// current document set to limit the processing of each new query
// token for efficiency (i.e., incremental intersection).
var queryTokenScores = {};
tokens.forEach(function (key) {
var docs = this.index[fieldName].getDocs(key);
var idf = this.idf(key, fieldName);
if (scores && booleanType == 'AND') {
// special case, we can rule out documents that have been
// already been filtered out because they weren't scored
// by previous query token passes.
var filteredDocs = {};
for (var docRef in scores) {
if (docRef in docs) {
filteredDocs[docRef] = docs[docRef];
}
}
docs = filteredDocs;
}
// only record appeared token for retrieved documents for the
// original token, not for expaned token.
// beause for doing coordNorm for a retrieved document, coordNorm only care how many
// query token appear in that document.
// so expanded token should not be added into docTokens, if added, this will pollute the
// coordNorm
if (key == token) {
this.fieldSearchStats(docTokens, key, docs);
}
for (var docRef in docs) {
var tf = this.index[fieldName].getTermFrequency(key, docRef);
var fieldLength = this.documentStore.getFieldLength(docRef, fieldName);
var fieldLengthNorm = 1;
if (fieldLength != 0) {
fieldLengthNorm = 1 / Math.sqrt(fieldLength);
}
var penality = 1;
if (key != token) {
// currently I'm not sure if this penality is enough,
// need to do verification
penality = (1 - (key.length - token.length) / key.length) * 0.15;
}
var score = tf * idf * fieldLengthNorm * penality;
if (docRef in queryTokenScores) {
queryTokenScores[docRef] += score;
} else {
queryTokenScores[docRef] = score;
}
}
}, this);
scores = this.mergeScores(scores, queryTokenScores, booleanType);
}, this);
scores = this.coordNorm(scores, docTokens, queryTokens.length);
return scores;
}...
var config = new elasticlunr.Configuration(configStr, this.getFields()).get();
var queryTokens = this.pipeline.run(elasticlunr.tokenizer(query));
var queryResults = {};
for (var field in config) {
var fieldSearchResults = this.fieldSearch(queryTokens, field, config);
var fieldBoost = config[field].boost;
for (var docRef in fieldSearchResults) {
fieldSearchResults[docRef] = fieldSearchResults[docRef] * fieldBoost;
}
for (var docRef in fieldSearchResults) {
...fieldSearchStats = function (docTokens, token, docs) {
for (var doc in docs) {
if (doc in docTokens) {
docTokens[doc].push(token);
} else {
docTokens[doc] = [token];
}
}
}...
// only record appeared token for retrieved documents for the
// original token, not for expaned token.
// beause for doing coordNorm for a retrieved document, coordNorm only care how many
// query token appear in that document.
// so expanded token should not be added into docTokens, if added, this will pollute the
// coordNorm
if (key == token) {
this.fieldSearchStats(docTokens, key, docs);
}
for (var docRef in docs) {
var tf = this.index[fieldName].getTermFrequency(key, docRef);
var fieldLength = this.documentStore.getFieldLength(docRef, fieldName);
var fieldLengthNorm = 1;
if (fieldLength != 0) {
...getFields = function () {
return this._fields.slice();
}...
if (!query) return [];
var configStr = null;
if (userConfig != null) {
configStr = JSON.stringify(userConfig);
}
var config = new elasticlunr.Configuration(configStr, this.getFields()).get();
var queryTokens = this.pipeline.run(elasticlunr.tokenizer(query));
var queryResults = {};
for (var field in config) {
var fieldSearchResults = this.fieldSearch(queryTokens, field, config);
...idf = function (term, field) {
var cacheKey = "@" + field + '/' + term;
if (Object.prototype.hasOwnProperty.call(this._idfCache, cacheKey)) return this._idfCache[cacheKey];
var df = this.index[field].getDocFreq(term);
var idf = 1 + Math.log(this.documentStore.length / (df + 1));
this._idfCache[cacheKey] = idf;
return idf;
}...
// query tokens can't add new documents to the result set, use the
// current document set to limit the processing of each new query
// token for efficiency (i.e., incremental intersection).
var queryTokenScores = {};
tokens.forEach(function (key) {
var docs = this.index[fieldName].getDocs(key);
var idf = this.idf(key, fieldName);
if (scores && booleanType == 'AND') {
// special case, we can rule out documents that have been
// already been filtered out because they weren't scored
// by previous query token passes.
var filteredDocs = {};
for (var docRef in scores) {
...mergeScores = function (accumScores, scores, op) {
if (!accumScores) {
return scores;
}
if (op == 'AND') {
var intersection = {};
for (var docRef in scores) {
if (docRef in accumScores) {
intersection[docRef] = accumScores[docRef] + scores[docRef];
}
}
return intersection;
} else {
for (var docRef in scores) {
if (docRef in accumScores) {
accumScores[docRef] += scores[docRef];
} else {
accumScores[docRef] = scores[docRef];
}
}
return accumScores;
}
}...
queryTokenScores[docRef] += score;
} else {
queryTokenScores[docRef] = score;
}
}
}, this);
scores = this.mergeScores(scores, queryTokenScores, booleanType);
}, this);
scores = this.coordNorm(scores, docTokens, queryTokens.length);
return scores;
};
/**
...off = function (name, fn) {
return this.eventEmitter.removeListener(name, fn);
}n/a
on = function () {
var args = Array.prototype.slice.call(arguments);
return this.eventEmitter.addListener.apply(this.eventEmitter, args);
}...
this._ref = 'id';
this.pipeline = new elasticlunr.Pipeline;
this.documentStore = new elasticlunr.DocumentStore;
this.index = {};
this.eventEmitter = new elasticlunr.EventEmitter;
this._idfCache = {};
this.on('add', 'remove', 'update', (function () {
this._idfCache = {};
}).bind(this));
};
/**
* Bind a handler to events being emitted by the index.
*
...removeDoc = function (doc, emitEvent) {
if (!doc) return;
var emitEvent = emitEvent === undefined ? true : emitEvent;
var docRef = doc[this._ref];
if (!this.documentStore.hasDoc(docRef)) return;
this.documentStore.removeDoc(docRef);
this._fields.forEach(function (field) {
var fieldTokens = this.pipeline.run(elasticlunr.tokenizer(doc[field]));
fieldTokens.forEach(function (token) {
this.index[field].removeToken(token, docRef);
}, this);
}, this);
if (emitEvent) this.eventEmitter.emit('remove', doc, this);
}...
index.addDoc(doc2);
```
If your JSON document contains field that not configured in index, then that field will not be indexed, which means that field is
not searchable.
## 3. Remove document from index
Elasticlunr.js support remove a document from index, just provide JSON document to <code>elasticlunr.Index.prototype.<
span class="apidocCodeKeywordSpan">removeDoc()</code> function.
For example:
```javascript
var doc = {
"id": 1,
"title": "Oracle released its latest database Oracle 12g",
"body": "Yestaday Oracle has released its new database Oracle 12g, this would make more money for this company and
lead to a nice profit report of annual year."
...removeDocByRef = function (docRef, emitEvent) {
if (!docRef) return;
if (this.documentStore.isDocStored() === false) {
return;
}
if (!this.documentStore.hasDoc(docRef)) return;
var doc = this.documentStore.getDoc(docRef);
this.removeDoc(doc, false);
}...
* @see Index.prototype.remove
* @see Index.prototype.add
* @memberOf Index
*/
elasticlunr.Index.prototype.updateDoc = function (doc, emitEvent) {
var emitEvent = emitEvent === undefined ? true : emitEvent;
this.removeDocByRef(doc[this._ref], false);
this.addDoc(doc, false);
if (emitEvent) this.eventEmitter.emit('update', doc, this);
};
/**
* Calculates the inverse document frequency for a token within the index of a field.
...saveDocument = function (save) {
this.documentStore = new elasticlunr.DocumentStore(save);
return this;
}...
If user do not want to store the original JSON documents, they could use the following setting:
```javascript
var index = elasticlunr(function () {
this.addField('title');
this.addField('body');
this.setRef('id');
this.saveDocument(false);
});
```
Then elasticlunr.js will not store the JSON documents, this will reduce the index size, but also bring some inconvenience such as
update a document or delete a document by document id or reference. Actually most of the time user will not udpate or delete a
document from index.
[API documentation](http://elasticlunr.com/docs/index.html) is available, as well as a [full working example](http://elasticlunr
.com/example/index.html).
...search = function (query, userConfig) {
if (!query) return [];
var configStr = null;
if (userConfig != null) {
configStr = JSON.stringify(userConfig);
}
var config = new elasticlunr.Configuration(configStr, this.getFields()).get();
var queryTokens = this.pipeline.run(elasticlunr.tokenizer(query));
var queryResults = {};
for (var field in config) {
var fieldSearchResults = this.fieldSearch(queryTokens, field, config);
var fieldBoost = config[field].boost;
for (var docRef in fieldSearchResults) {
fieldSearchResults[docRef] = fieldSearchResults[docRef] * fieldBoost;
}
for (var docRef in fieldSearchResults) {
if (docRef in queryResults) {
queryResults[docRef] += fieldSearchResults[docRef];
} else {
queryResults[docRef] = fieldSearchResults[docRef];
}
}
}
var results = [];
for (var docRef in queryResults) {
results.push({ref: docRef, score: queryResults[docRef]});
}
results.sort(function (a, b) { return b.score - a.score; });
return results;
}...
index.addDoc(doc1);
index.addDoc(doc2);
```
Then searching is as simple:
```javascript
index.search("Oracle database profit");
```
Also, you could do query-time boosting by passing in a configuration.
```javascript
index.search("Oracle database profit", {
fields: {
...setRef = function (refName) {
this._ref = refName;
return this;
}...
A very simple search index can be created using the following scripts:
```javascript
var index = elasticlunr(function () {
this.addField('title');
this.addField('body');
this.setRef('id');
});
```
Adding documents to the index is as simple as:
```javascript
var doc1 = {
...toJSON = function () {
var indexJson = {};
this._fields.forEach(function (field) {
indexJson[field] = this.index[field].toJSON();
}, this);
return {
version: elasticlunr.version,
fields: this._fields,
ref: this._ref,
documentStore: this.documentStore.toJSON(),
index: indexJson,
pipeline: this.pipeline.toJSON()
};
}...
*
* @return {Object}
* @memberOf Index
*/
elasticlunr.Index.prototype.toJSON = function () {
var indexJson = {};
this._fields.forEach(function (field) {
indexJson[field] = this.index[field].toJSON();
}, this);
return {
version: elasticlunr.version,
fields: this._fields,
ref: this._ref,
documentStore: this.documentStore.toJSON(),
...updateDoc = function (doc, emitEvent) {
var emitEvent = emitEvent === undefined ? true : emitEvent;
this.removeDocByRef(doc[this._ref], false);
this.addDoc(doc, false);
if (emitEvent) this.eventEmitter.emit('update', doc, this);
}n/a
use = function (plugin) {
var args = Array.prototype.slice.call(arguments, 1);
args.unshift(this);
plugin.apply(this, args);
}...
<script src="lunr.de.js"></script>
```
then, you could use elasticlunr.js as normal:
```javascript
var index = elasticlunr(function () {
// use the language (de)
this.use(elasticlunr.de);
// then, the normal elasticlunr index initialization
this.addField('title')
this.addField('body')
});
```
Pay attention to the special code:
...InvertedIndex = function () {
this.root = { docs: {}, df: 0 };
}n/a
load = function (serialisedData) {
var idx = new this;
idx.root = serialisedData.root;
return idx;
}...
+ elasticlunr.version + ' importing ' + serialisedData.version);
}
var idx = new this;
idx._fields = serialisedData.fields;
idx._ref = serialisedData.ref;
idx.documentStore = elasticlunr.DocumentStore.load(serialisedData.documentStore);
idx.pipeline = elasticlunr.Pipeline.load(serialisedData.pipeline);
idx.index = {};
for (var field in serialisedData.index) {
idx.index[field] = elasticlunr.InvertedIndex.load(serialisedData.index[field]);
}
return idx;
...addToken = function (token, tokenInfo, root) {
var root = root || this.root,
idx = 0;
while (idx <= token.length - 1) {
var key = token[idx];
if (!(key in root)) root[key] = {docs: {}, df: 0};
idx += 1;
root = root[key];
}
var docRef = tokenInfo.ref;
if (!root.docs[docRef]) {
// if this doc not exist, then add this doc
root.docs[docRef] = {tf: tokenInfo.tf};
root.df += 1;
} else {
// if this doc already exist, then update tokenInfo
root.docs[docRef] = {tf: tokenInfo.tf};
}
}...
if (token in tokenCount) tokenCount[token] += 1;
else tokenCount[token] = 1;
}, this);
for (var token in tokenCount) {
var termFrequency = tokenCount[token];
termFrequency = Math.sqrt(termFrequency);
this.index[field].addToken(token, { ref: docRef, tf: termFrequency });
}
}, this);
if (emitEvent) this.eventEmitter.emit('add', doc, this);
};
/**
...expandToken = function (token, memo, root) {
if (token == null || token == '') return [];
var memo = memo || [];
if (root == void 0) {
root = this.getNode(token);
if (root == null) return memo;
}
if (root.df > 0) memo.push(token);
for (var key in root) {
if (key === 'docs') continue;
if (key === 'df') continue;
this.expandToken(token + key, memo, root[key]);
}
return memo;
}...
if (boost === 0) {
return;
}
queryTokens.forEach(function (token) {
var tokens = [token];
if (expand == true) {
tokens = this.index[fieldName].expandToken(token);
}
// Consider every query token in turn. If expanded, each query token
// corresponds to a set of tokens, which is all tokens in the
// index matching the pattern queryToken* .
// For the set of tokens corresponding to a query token, find and score
// all matching documents. Store those scores in queryTokenScores,
// keyed by docRef.
...getDocFreq = function (token) {
var node = this.getNode(token);
if (node == null) {
return 0;
}
return node.df;
}...
* @private
* @memberOf Index
*/
elasticlunr.Index.prototype.idf = function (term, field) {
var cacheKey = "@" + field + '/' + term;
if (Object.prototype.hasOwnProperty.call(this._idfCache, cacheKey)) return this._idfCache[cacheKey];
var df = this.index[field].getDocFreq(term);
var idf = 1 + Math.log(this.documentStore.length / (df + 1));
this._idfCache[cacheKey] = idf;
return idf;
};
/**
...getDocs = function (token) {
var node = this.getNode(token);
if (node == null) {
return {};
}
return node.docs;
}...
// Furthermore, since when booleanType is AND, additional
// query tokens can't add new documents to the result set, use the
// current document set to limit the processing of each new query
// token for efficiency (i.e., incremental intersection).
var queryTokenScores = {};
tokens.forEach(function (key) {
var docs = this.index[fieldName].getDocs(key);
var idf = this.idf(key, fieldName);
if (scores && booleanType == 'AND') {
// special case, we can rule out documents that have been
// already been filtered out because they weren't scored
// by previous query token passes.
var filteredDocs = {};
...getNode = function (token) {
if (!token) return null;
var node = this.root;
for (var i = 0; i < token.length; i++) {
if (!node[token[i]]) return null;
node = node[token[i]];
}
return node;
}...
*
*
* @param {String} token The token to get the documents for.
* @return {Object}
* @memberOf InvertedIndex
*/
elasticlunr.InvertedIndex.prototype.getDocs = function (token) {
var node = this.getNode(token);
if (node == null) {
return {};
}
return node.docs;
};
...getTermFrequency = function (token, docRef) {
var node = this.getNode(token);
if (node == null) {
return 0;
}
if (!(docRef in node.docs)) {
return 0;
}
return node.docs[docRef].tf;
}...
// so expanded token should not be added into docTokens, if added, this will pollute the
// coordNorm
if (key == token) {
this.fieldSearchStats(docTokens, key, docs);
}
for (var docRef in docs) {
var tf = this.index[fieldName].getTermFrequency(key, docRef);
var fieldLength = this.documentStore.getFieldLength(docRef, fieldName);
var fieldLengthNorm = 1;
if (fieldLength != 0) {
fieldLengthNorm = 1 / Math.sqrt(fieldLength);
}
var penality = 1;
...hasToken = function (token) {
if (!token) return false;
var node = this.root;
for (var i = 0; i < token.length; i++) {
if (!node[token[i]]) return false;
node = node[token[i]];
}
return true;
}n/a
removeToken = function (token, ref) {
if (!token) return;
var node = this.getNode(token);
if (node == null) return;
if (ref in node.docs) {
delete node.docs[ref];
node.df -= 1;
}
}...
if (!this.documentStore.hasDoc(docRef)) return;
this.documentStore.removeDoc(docRef);
this._fields.forEach(function (field) {
var fieldTokens = this.pipeline.run(elasticlunr.tokenizer(doc[field]));
fieldTokens.forEach(function (token) {
this.index[field].removeToken(token, docRef);
}, this);
}, this);
if (emitEvent) this.eventEmitter.emit('remove', doc, this);
};
/**
...toJSON = function () {
return {
root: this.root
};
}...
*
* @return {Object}
* @memberOf Index
*/
elasticlunr.Index.prototype.toJSON = function () {
var indexJson = {};
this._fields.forEach(function (field) {
indexJson[field] = this.index[field].toJSON();
}, this);
return {
version: elasticlunr.version,
fields: this._fields,
ref: this._ref,
documentStore: this.documentStore.toJSON(),
...Pipeline = function () {
this._queue = [];
}n/a
getRegisteredFunction = function (label) {
if ((label in elasticlunr.Pipeline.registeredFunctions) !== true) {
return null;
}
return elasticlunr.Pipeline.registeredFunctions[label];
}...
* @return {elasticlunr.Pipeline}
* @memberOf Pipeline
*/
elasticlunr.Pipeline.load = function (serialised) {
var pipeline = new elasticlunr.Pipeline;
serialised.forEach(function (fnName) {
var fn = elasticlunr.Pipeline.getRegisteredFunction(fnName);
if (fn) {
pipeline.add(fn);
} else {
throw new Error('Cannot load un-registered function: ' + fnName);
}
});
...load = function (serialised) {
var pipeline = new elasticlunr.Pipeline;
serialised.forEach(function (fnName) {
var fn = elasticlunr.Pipeline.getRegisteredFunction(fnName);
if (fn) {
pipeline.add(fn);
} else {
throw new Error('Cannot load un-registered function: ' + fnName);
}
});
return pipeline;
}...
+ elasticlunr.version + ' importing ' + serialisedData.version);
}
var idx = new this;
idx._fields = serialisedData.fields;
idx._ref = serialisedData.ref;
idx.documentStore = elasticlunr.DocumentStore.load(serialisedData.documentStore);
idx.pipeline = elasticlunr.Pipeline.load(serialisedData.pipeline);
idx.index = {};
for (var field in serialisedData.index) {
idx.index[field] = elasticlunr.InvertedIndex.load(serialisedData.index[field]);
}
return idx;
...registerFunction = function (fn, label) {
if (label in elasticlunr.Pipeline.registeredFunctions) {
elasticlunr.utils.warn('Overwriting existing registered function: ' + label);
}
fn.label = label;
elasticlunr.Pipeline.registeredFunctions[label] = fn;
}...
return w;
};
return porterStemmer;
})();
elasticlunr.Pipeline.registerFunction(elasticlunr.stemmer, 'stemmer');
/*!
* elasticlunr.stopWordFilter
* Copyright (C) 2016 Oliver Nightingale
* Copyright (C) 2016 Wei Song
*/
/**
...warnIfFunctionNotRegistered = function (fn) {
var isRegistered = fn.label && (fn.label in this.registeredFunctions);
if (!isRegistered) {
elasticlunr.utils.warn('Function is not registered with pipeline. This may cause problems when serialising the index.\n', fn
);
}
}...
* @param {Function} functions Any number of functions to add to the pipeline.
* @memberOf Pipeline
*/
elasticlunr.Pipeline.prototype.add = function () {
var fns = Array.prototype.slice.call(arguments);
fns.forEach(function (fn) {
elasticlunr.Pipeline.warnIfFunctionNotRegistered(fn);
this._queue.push(fn);
}, this);
};
/**
* Adds a single function after a function that already exists in the
* pipeline.
...add = function () {
var fns = Array.prototype.slice.call(arguments);
fns.forEach(function (fn) {
elasticlunr.Pipeline.warnIfFunctionNotRegistered(fn);
this._queue.push(fn);
}, this);
}...
* var idx = elasticlunr(function () {
* this.addField('id');
* this.addField('title');
* this.addField('body');
*
* //this.setRef('id'); // default ref is 'id'
*
* this.pipeline.add(function () {
* // some custom pipeline function
* });
* });
*
* idx.addDoc({
* id: 1,
* title: 'Oracle released database 12g',
...after = function (existingFn, newFn) {
elasticlunr.Pipeline.warnIfFunctionNotRegistered(newFn);
var pos = this._queue.indexOf(existingFn);
if (pos === -1) {
throw new Error('Cannot find existingFn');
}
this._queue.splice(pos + 1, 0, newFn);
}n/a
before = function (existingFn, newFn) {
elasticlunr.Pipeline.warnIfFunctionNotRegistered(newFn);
var pos = this._queue.indexOf(existingFn);
if (pos === -1) {
throw new Error('Cannot find existingFn');
}
this._queue.splice(pos, 0, newFn);
}n/a
get = function () {
return this._queue;
}...
if (!query) return [];
var configStr = null;
if (userConfig != null) {
configStr = JSON.stringify(userConfig);
}
var config = new elasticlunr.Configuration(configStr, this.getFields()).get();
var queryTokens = this.pipeline.run(elasticlunr.tokenizer(query));
var queryResults = {};
for (var field in config) {
var fieldSearchResults = this.fieldSearch(queryTokens, field, config);
...remove = function (fn) {
var pos = this._queue.indexOf(fn);
if (pos === -1) {
return;
}
this._queue.splice(pos, 1);
}n/a
reset = function () {
this._queue = [];
}...
/**
* Build default search configuration.
*
* @param {Array} fields fields of index instance
*/
elasticlunr.Configuration.prototype.buildDefaultConfig = function (fields) {
this.reset();
fields.forEach(function (field) {
this.config[field] = {
boost: 1,
bool: "OR",
expand: false
};
}, this);
...run = function (tokens) {
var out = [],
tokenLength = tokens.length,
pipelineLength = this._queue.length;
for (var i = 0; i < tokenLength; i++) {
var token = tokens[i];
for (var j = 0; j < pipelineLength; j++) {
token = this._queue[j](token, i, tokens);
if (token === void 0 || token === null) break;
};
if (token !== void 0 && token !== null) out.push(token);
};
return out;
}...
if (!doc) return;
var emitEvent = emitEvent === undefined ? true : emitEvent;
var docRef = doc[this._ref];
this.documentStore.addDoc(docRef, doc);
this._fields.forEach(function (field) {
var fieldTokens = this.pipeline.run(elasticlunr.tokenizer(doc[field]));
this.documentStore.addFieldLength(docRef, field, fieldTokens.length);
var tokenCount = {};
fieldTokens.forEach(function (token) {
if (token in tokenCount) tokenCount[token] += 1;
else tokenCount[token] = 1;
}, this);
...toJSON = function () {
return this._queue.map(function (fn) {
elasticlunr.Pipeline.warnIfFunctionNotRegistered(fn);
return fn.label;
});
}...
*
* @return {Object}
* @memberOf Index
*/
elasticlunr.Index.prototype.toJSON = function () {
var indexJson = {};
this._fields.forEach(function (field) {
indexJson[field] = this.index[field].toJSON();
}, this);
return {
version: elasticlunr.version,
fields: this._fields,
ref: this._ref,
documentStore: this.documentStore.toJSON(),
...function porterStemmer(w) {
var stem,
suffix,
firstch,
re,
re2,
re3,
re4;
if (w.length < 3) { return w; }
firstch = w.substr(0,1);
if (firstch == "y") {
w = firstch.toUpperCase() + w.substr(1);
}
// Step 1a
re = re_1a
re2 = re2_1a;
if (re.test(w)) { w = w.replace(re,"$1$2"); }
else if (re2.test(w)) { w = w.replace(re2,"$1$2"); }
// Step 1b
re = re_1b;
re2 = re2_1b;
if (re.test(w)) {
var fp = re.exec(w);
re = re_mgr0;
if (re.test(fp[1])) {
re = re_1b_2;
w = w.replace(re,"");
}
} else if (re2.test(w)) {
var fp = re2.exec(w);
stem = fp[1];
re2 = re_s_v;
if (re2.test(stem)) {
w = stem;
re2 = re2_1b_2;
re3 = re3_1b_2;
re4 = re4_1b_2;
if (re2.test(w)) { w = w + "e"; }
else if (re3.test(w)) { re = re_1b_2; w = w.replace(re,""); }
else if (re4.test(w)) { w = w + "e"; }
}
}
// Step 1c - replace suffix y or Y by i if preceded by a non-vowel which is not the first letter of the word (so cry -> cri, by
-> by, say -> say)
re = re_1c;
if (re.test(w)) {
var fp = re.exec(w);
stem = fp[1];
w = stem + "i";
}
// Step 2
re = re_2;
if (re.test(w)) {
var fp = re.exec(w);
stem = fp[1];
suffix = fp[2];
re = re_mgr0;
if (re.test(stem)) {
w = stem + step2list[suffix];
}
}
// Step 3
re = re_3;
if (re.test(w)) {
var fp = re.exec(w);
stem = fp[1];
suffix = fp[2];
re = re_mgr0;
if (re.test(stem)) {
w = stem + step3list[suffix];
}
}
// Step 4
re = re_4;
re2 = re2_4;
if (re.test(w)) {
var fp = re.exec(w);
stem = fp[1];
re = re_mgr1;
if (re.test(stem)) {
w = stem;
}
} else if (re2.test(w)) {
var fp = re2.exec(w);
stem = fp[1] + fp[2];
re2 = re_mgr1;
if (re2.test(stem)) {
w = stem;
}
}
// Step 5
re = re_5;
if (re.test(w)) {
var fp = re.exec(w);
stem = fp[1];
re = re_mgr1;
re2 = re_meq1;
re3 = re3_5;
if (re.test(stem) || (re2.test(stem) && !(re3.test(stem)))) {
w = stem;
}
}
re = re_5_1;
re2 = re_mgr1;
if (re.test(w) && re2.test(w)) {
re = re_1b_2;
w = w.replace(re,"");
}
// and turn initial Y back to y
if (firstch == "y") {
w = firstch.toLowerCase() + w.substr(1);
}
return w;
}n/a
stopWordFilter = function (token) {
if (token && elasticlunr.stopWordFilter.stopWords[token] !== true) {
return token;
}
}n/a
trimmer = function (token) {
if (token === null || token === undefined) {
throw new Error('token should not be undefined');
}
return token
.replace(/^\W+/, '')
.replace(/\W+$/, '');
}n/a
SortedSet = function () {
this.length = 0
this.elements = []
}n/a
load = function (serialisedData) {
var set = new this
set.elements = serialisedData
set.length = serialisedData.length
return set
}...
+ elasticlunr.version + ' importing ' + serialisedData.version);
}
var idx = new this;
idx._fields = serialisedData.fields;
idx._ref = serialisedData.ref;
idx.documentStore = elasticlunr.DocumentStore.load(serialisedData.documentStore);
idx.pipeline = elasticlunr.Pipeline.load(serialisedData.pipeline);
idx.index = {};
for (var field in serialisedData.index) {
idx.index[field] = elasticlunr.InvertedIndex.load(serialisedData.index[field]);
}
return idx;
...add = function () {
var i, element
for (i = 0; i < arguments.length; i++) {
element = arguments[i]
if (~this.indexOf(element)) continue
this.elements.splice(this.locationFor(element), 0, element)
}
this.length = this.elements.length
}...
* var idx = elasticlunr(function () {
* this.addField('id');
* this.addField('title');
* this.addField('body');
*
* //this.setRef('id'); // default ref is 'id'
*
* this.pipeline.add(function () {
* // some custom pipeline function
* });
* });
*
* idx.addDoc({
* id: 1,
* title: 'Oracle released database 12g',
...clone = function () {
var clone = new lunr.SortedSet
clone.elements = this.toArray()
clone.length = clone.elements.length
return clone
}...
if (this.length >= otherSet.length) {
longSet = this, shortSet = otherSet
} else {
longSet = otherSet, shortSet = this
}
unionSet = longSet.clone()
for(var i = 0, shortSetElements = shortSet.toArray(); i < shortSetElements.length; i++){
unionSet.add(shortSetElements[i])
}
return unionSet
}
...forEach = function (fn, ctx) {
return this.elements.forEach(fn, ctx)
}...
elasticlunr.EventEmitter.prototype.addListener = function () {
var args = Array.prototype.slice.call(arguments),
fn = args.pop(),
names = args;
if (typeof fn !== "function") throw new TypeError ("last argument must be a function");
names.forEach(function (name) {
if (!this.hasHandler(name)) this.events[name] = [];
this.events[name].push(fn);
}, this);
};
/**
* Removes a handler function from a specific event.
...indexOf = function (elem) {
var start = 0,
end = this.elements.length,
sectionLength = end - start,
pivot = start + Math.floor(sectionLength / 2),
pivotElem = this.elements[pivot]
while (sectionLength > 1) {
if (pivotElem === elem) return pivot
if (pivotElem < elem) start = pivot
if (pivotElem > elem) end = pivot
sectionLength = end - start
pivot = start + Math.floor(sectionLength / 2)
pivotElem = this.elements[pivot]
}
if (pivotElem === elem) return pivot
return -1
}...
* @param {String} eventName The name of the event to remove this function from.
* @param {Function} fn The function to remove from an event.
* @memberOf EventEmitter
*/
elasticlunr.EventEmitter.prototype.removeListener = function (name, fn) {
if (!this.hasHandler(name)) return;
var fnIndex = this.events[name].indexOf(fn);
if (fnIndex === -1) return;
this.events[name].splice(fnIndex, 1);
if (this.events[name].length == 0) delete this.events[name];
};
...intersect = function (otherSet) {
var intersectSet = new lunr.SortedSet,
i = 0, j = 0,
a_len = this.length, b_len = otherSet.length,
a = this.elements, b = otherSet.elements
while (true) {
if (i > a_len - 1 || j > b_len - 1) break
if (a[i] === b[j]) {
intersectSet.add(a[i])
i++, j++
continue
}
if (a[i] < b[j]) {
i++
continue
}
if (a[i] > b[j]) {
j++
continue
}
};
return intersectSet
}n/a
locationFor = function (elem) {
var start = 0,
end = this.elements.length,
sectionLength = end - start,
pivot = start + Math.floor(sectionLength / 2),
pivotElem = this.elements[pivot]
while (sectionLength > 1) {
if (pivotElem < elem) start = pivot
if (pivotElem > elem) end = pivot
sectionLength = end - start
pivot = start + Math.floor(sectionLength / 2)
pivotElem = this.elements[pivot]
}
if (pivotElem > elem) return pivot
if (pivotElem < elem) return pivot + 1
}...
*/
lunr.SortedSet.prototype.add = function () {
var i, element
for (i = 0; i < arguments.length; i++) {
element = arguments[i]
if (~this.indexOf(element)) continue
this.elements.splice(this.locationFor(element), 0, element)
}
this.length = this.elements.length
}
/**
* Converts this sorted set into an array.
...map = function (fn, ctx) {
return this.elements.map(fn, ctx)
}...
if (token === null || token === undefined) {
return false;
}
return true;
});
arr = arr.map(function (t) {
return elasticlunr.utils.toString(t).toLowerCase();
});
var out = [];
arr.forEach(function(item) {
var tokens = item.split(elasticlunr.tokenizer.seperator);
out = out.concat(tokens);
...toArray = function () {
return this.elements.slice()
}...
*
* @returns {lunr.SortedSet}
* @memberOf SortedSet
*/
lunr.SortedSet.prototype.clone = function () {
var clone = new lunr.SortedSet
clone.elements = this.toArray()
clone.length = clone.elements.length
return clone
}
/**
* Creates a new lunr.SortedSet that contains the elements in the union
...toJSON = function () {
return this.toArray()
}...
*
* @return {Object}
* @memberOf Index
*/
elasticlunr.Index.prototype.toJSON = function () {
var indexJson = {};
this._fields.forEach(function (field) {
indexJson[field] = this.index[field].toJSON();
}, this);
return {
version: elasticlunr.version,
fields: this._fields,
ref: this._ref,
documentStore: this.documentStore.toJSON(),
...union = function (otherSet) {
var longSet, shortSet, unionSet
if (this.length >= otherSet.length) {
longSet = this, shortSet = otherSet
} else {
longSet = otherSet, shortSet = this
}
unionSet = longSet.clone()
for(var i = 0, shortSetElements = shortSet.toArray(); i < shortSetElements.length; i++){
unionSet.add(shortSetElements[i])
}
return unionSet
}n/a
tokenizer = function (str) {
if (!arguments.length || str === null || str === undefined) return [];
if (Array.isArray(str)) {
var arr = str.filter(function(token) {
if (token === null || token === undefined) {
return false;
}
return true;
});
arr = arr.map(function (t) {
return elasticlunr.utils.toString(t).toLowerCase();
});
var out = [];
arr.forEach(function(item) {
var tokens = item.split(elasticlunr.tokenizer.seperator);
out = out.concat(tokens);
}, this);
return out;
}
return str.toString().trim().toLowerCase().split(elasticlunr.tokenizer.seperator);
}...
if (!doc) return;
var emitEvent = emitEvent === undefined ? true : emitEvent;
var docRef = doc[this._ref];
this.documentStore.addDoc(docRef, doc);
this._fields.forEach(function (field) {
var fieldTokens = this.pipeline.run(elasticlunr.tokenizer(doc[field]));
this.documentStore.addFieldLength(docRef, field, fieldTokens.length);
var tokenCount = {};
fieldTokens.forEach(function (token) {
if (token in tokenCount) tokenCount[token] += 1;
else tokenCount[token] = 1;
}, this);
...getSeperator = function () {
return elasticlunr.tokenizer.seperator;
}n/a
resetSeperator = function () {
elasticlunr.tokenizer.seperator = elasticlunr.tokenizer.defaultSeperator;
}n/a
setSeperator = function (sep) {
if (sep !== null && sep !== undefined && typeof(sep) === 'object') {
elasticlunr.tokenizer.seperator = sep;
}
}n/a
toString = function (obj) {
if (obj === void 0 || obj === null) {
return "";
}
return obj.toString();
}...
* @memberOf Utils
*/
elasticlunr.utils.toString = function (obj) {
if (obj === void 0 || obj === null) {
return "";
}
return obj.toString();
};
/*!
* elasticlunr.EventEmitter
* Copyright (C) 2016 Oliver Nightingale
* Copyright (C) 2016 Wei Song
*/
...warn = function (message) {
if (global.console && console.warn) {
console.warn(message);
}
}...
*
* @param {String} message The message to be printed.
* @memberOf Utils
*/
elasticlunr.utils.warn = (function (global) {
return function (message) {
if (global.console && console.warn) {
console.warn(message);
}
};
})(this);
/**
* Convert an object to string.
*
...