private static class MyExpertScriptEngine implements ScriptEngine {
//可以命名自己在脚本api中使用的名称来引用这个脚本后端。
@Override
public String getType() {
return "expert_scripts";
}
//核心方法，下面是通过java的lamada表达式来实现的
@Override
public T compile(String scriptName, String scriptSource, ScriptContext context, Map params) {
if (context.equals(SearchScript.CONTEXT) == false) {
throw new IllegalArgumentException(getType() + " scripts cannot be used for context [" + context.name + "]");
}
// we use the script "source" as the script identifier
if ("pure_df".equals(scriptSource)) {
//通过p来获取参数params中的值，lookup得到文档中的的值
SearchScript.Factory factory = (p, lookup) -> new SearchScript.LeafFactory() {
final String field;
final String term;
{
if (p.containsKey("field") == false) {
throw new IllegalArgumentException("Missing parameter [field]");
}
if (p.containsKey("term") == false) {
throw new IllegalArgumentException("Missing parameter [term]");
}
field = p.get("field").toString();
term = p.get("term").toString();
}
@Override
public SearchScript newInstance(LeafReaderContext context) throws IOException {
PostingsEnum postings = context.reader().postings(new Term(field, term));
if (postings == null) {
// the field and/or term don't exist in this segment, so always return 0
return new SearchScript(p, lookup, context) {
@Override
public double runAsDouble() {
return 0.0d;
}
};
}
return new SearchScript(p, lookup, context) {
int currentDocid = -1;
@Override
public void setDocument(int docid) {
// advance has undefined behavior calling with a docid <= its current docid
if (postings.docID() < docid) {
try {
postings.advance(docid);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
currentDocid = docid;
}
@Override
public double runAsDouble() {
if (postings.docID() != currentDocid) {
// advance moved past the current doc, so this doc has no occurrences of the term
return 0.0d;
}
try {
return postings.freq();
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
};
}
@Override
public boolean needs_score() {
return false;
}
};
return context.factoryClazz.cast(factory);
}
throw new IllegalArgumentException("Unknown script name " + scriptSource);
}
@Override
public void close() {
// optionally close resources
}
}
通过分析上面的代码，我们给出自己的脚步：
开发自己的脚步v1
package com;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.index.LeafReaderContext;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.ScriptEngine;
import org.elasticsearch.script.SearchScript;
import java.io.IOException;
import java.util.*;
/**
* \* Created with IntelliJ IDEA.
* \* User: 王火斌
* \* Date: 18-8-９
* \* Time: 下午2:32
* \* Description:为了得到个性化推荐搜索效果，我们计算用户向量与每个产品特征向量的相似度。
* 相似度越高，最后得到的分值越高，排序越靠前.
* \
*/
public class FeatureVectorScoreSearchScript implements ScriptEngine {
private final static Logger logger = LogManager.getLogger(FeatureVectorScoreSearchScript.class);
@Override
public String getType() {
return "feature_vector_scoring_script";
}
@Override
public T compile(String scriptName, String scriptSource, ScriptContext context, Map params) {
logger.info("The feature_vector_scoring_script is calculating the similarity of users and commodities");
if (!context.equals(SearchScript.CONTEXT)) {
throw new IllegalArgumentException(getType() + " scripts cannot be used for context [" + context.name + "]");
}
if("whb_fvs".equals(scriptSource)) {
SearchScript.Factory factory = (p, lookup) -> new SearchScript.LeafFactory() {
// 对入参检查
final Map inputFeatureVector;
final String field;
{
if (p.containsKey("field") == false) {
throw new IllegalArgumentException("Missing parameter [field]");
}
if(p.containsKey("inputFeatureVector") == false){
throw new IllegalArgumentException("Missing parameter [inputFeatureVector]");
}
field = p.get("field").toString();
inputFeatureVector = (Map) p.get("inputFeatureVector");
}
@Override
public SearchScript newInstance(LeafReaderContext context) throws IOException {
return new SearchScript(p, lookup, context) {
@Override
public double runAsDouble() {
if(lookup.source().containsKey(field)==true){
final Map productFeatureVector = (Map) lookup.source().get(field);
return calculateVectorSimilarity(inputFeatureVector, productFeatureVector);
}else {
logger.info("The " + field + " is not exist in the product");
return 0.0D;
}
}
};
}
@Override
public boolean needs_score() {
return false;
}
};
return context.factoryClazz.cast(factory);
}throw new IllegalArgumentException("Unknown script name " + scriptSource);
}
@Override
public void close() {
}
//计算两个向量的相似度(cos)
public double calculateVectorSimilarity(Map inputFeatureVector , Map productFeatureVector){
double sumOfProduct = 0.0D;
double sumOfUser = 0.0D;
double sumOfSquare = 0.0D;
if(inputFeatureVector!=null && productFeatureVector!=null){
for(Map.Entry entry: inputFeatureVector.entrySet()){
String dimName = entry.getKey();
double dimScore = Double.parseDouble(entry.getValue().toString());
double itemDimScore = productFeatureVector.get(dimName);
sumOfUser += dimScore*dimScore;
sumOfProduct += itemDimScore*itemDimScore;
sumOfSquare += dimScore*itemDimScore;
}
if(sumOfUser*sumOfProduct==0.0D){
return 0.0D;
}
return sumOfSquare / (Math.sqrt(sumOfUser)*Math.sqrt(sumOfProduct));
}else {
return 0.0D;
}
}
}
fast-vector-distance
/**
* \* Created with IntelliJ IDEA.
* \* User: 王火斌
* \* Date: 18-8-９
* \* Time: 下午2:32
* \* Description:为了得到个性化推荐搜索效果，我们计算用户向量与每个产品特征向量的相似度。
* 相似度越高，最后得到的分值越高，排序越靠前.
* \
*/
/**
package com;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.index.LeafReaderContext;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.plugins.ScriptPlugin;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.ScriptEngine;
import org.elasticsearch.script.SearchScript;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.store.ByteArrayDataInput;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.DoubleBuffer;
import java.util.*;
* This class is instantiated when Elasticsearch loads the plugin for the
* first time. If you change the name of this plugin, make sure to update
* src/main/resources/es-plugin.properties file that points to this class.
*/
public final class FastVectorDistance extends Plugin implements ScriptPlugin {
@Override
public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) {
return new FastVectorDistanceEngine();
}
private static class FastVectorDistanceEngine implements ScriptEngine {
private final static Logger logger = LogManager.getLogger(FastVectorDistance.class);
private static final int DOUBLE_SIZE = 8;
double queryVectorNorm;
@Override
public String getType() {
return "feature_vector_scoring_script";
}
@Override
public T compile(String scriptName, String scriptSource, ScriptContext context, Map params) {
logger.info("The feature_vector_scoring_script is calculating the similarity of users and commodities");
if (!context.equals(SearchScript.CONTEXT)) {
throw new IllegalArgumentException(getType() + " scripts cannot be used for context [" + context.name + "]");
}
if ("whb_fvd".equals(scriptSource)) {
SearchScript.Factory factory = (p, lookup) -> new SearchScript.LeafFactory() {
// The field to compare against
final String field;
//Whether this search should be cosine or dot product
final Boolean cosine;
//The query embedded vector
final Object vector;
Boolean exclude;
//The final comma delimited vector representation of the query vector
double[] inputVector;
{
if (p.containsKey("field") == false) {
throw new IllegalArgumentException("Missing parameter [field]");
}
//Determine if cosine
final Object cosineBool = p.get("cosine");
cosine = cosineBool != null ? (boolean) cosineBool : true;
//Get the field value from the query
field = p.get("field").toString();
final Object excludeBool = p.get("exclude");
exclude = excludeBool != null ? (boolean) cosineBool : true;
//Get the query vector embedding
vector = p.get("vector");
//Determine if raw comma-delimited vector or embedding was passed
if (vector != null) {
final ArrayList tmp = (ArrayList) vector;
inputVector = new double[tmp.size()];
for (int i = 0; i < inputVector.length; i++) {
inputVector[i] = tmp.get(i);
}
} else {
final Object encodedVector = p.get("encoded_vector");
if (encodedVector == null) {
throw new IllegalArgumentException("Must have 'vector' or 'encoded_vector' as a parameter");
}
inputVector = Util.convertBase64ToArray((String) encodedVector);
}
//If cosine calculate the query vec norm
if (cosine) {
queryVectorNorm = 0d;
// compute query inputVector norm once
for (double v : inputVector) {
queryVectorNorm += Math.pow(v, 2.0);
}
}
}
@Override
public SearchScript newInstance(LeafReaderContext context) throws IOException {
return new SearchScript(p, lookup, context) {
Boolean is_value = false;
// Use Lucene LeafReadContext to access binary values directly.
BinaryDocValues accessor = context.reader().getBinaryDocValues(field);
@Override
public void setDocument(int docId) {
// advance has undefined behavior calling with a docid <= its current docid
try {
accessor.advanceExact(docId);
is_value = true;
} catch (IOException e) {
is_value = false;
}
}
@Override
public double runAsDouble() {
//If there is no field value return 0 rather than fail.
if (!is_value) return 0.0d;
final int inputVectorSize = inputVector.length;
final byte[] bytes;
try {
bytes = accessor.binaryValue().bytes;
} catch (IOException e) {
return 0d;
}
final ByteArrayDataInput byteDocVector = new ByteArrayDataInput(bytes);
byteDocVector.readVInt();
final int docVectorLength = byteDocVector.readVInt(); // returns the number of bytes to read
if (docVectorLength != inputVectorSize * DOUBLE_SIZE) {
return 0d;
}
final int position = byteDocVector.getPosition();
final DoubleBuffer doubleBuffer = ByteBuffer.wrap(bytes, position, docVectorLength).asDoubleBuffer();
final double[] docVector = new double[inputVectorSize];
doubleBuffer.get(docVector);
double docVectorNorm = 0d;
double score = 0d;
//calculate dot product of document vector and query vector
for (int i = 0; i < inputVectorSize; i++) {
score += docVector[i] * inputVector[i];
if (cosine) {
docVectorNorm += Math.pow(docVector[i], 2.0);
}
}
//If cosine, calcluate cosine score
if (cosine) {
if (docVectorNorm == 0 || queryVectorNorm == 0) return 0d;
score = score / (Math.sqrt(docVectorNorm) * Math.sqrt(queryVectorNorm));
}
return score;
}
};
}
@Override
public boolean needs_score() {
return false;
}
};
return context.factoryClazz.cast(factory);
}
throw new IllegalArgumentException("Unknown script name " + scriptSource);
}
@Override
public void close() {}
}
}
description=feature-vector-similarity
version=1.0
name=feature-vector-score
site=${elasticsearch.plugin.site}
jvm=true
classname=com.FeatureVectorScoreSearchPlugin
java.version=1.8
elasticsearch.version=6.1.1
isolated=${elasticsearch.plugin.isolated}
description:feature-vector-similarity of the plugin
version(String):plugin’s version
name(String):the plugin name
classname(String):the name of the class to load, fully-qualified.
java.version(String):version of java the code is built against. Use the system property java.specification.version. Version string must be a sequence of nonnegative decimal integers separated by
"."'s and may have leading zeros.
测试
创建索引
create_index = {
"settings": {
"analysis": {
"analyzer": {
# this configures the custom analyzer we need to parse vectors such that the scoring
# plugin will work correctly
"payload_analyzer": {
"type": "custom",
"tokenizer":"whitespace",
"filter":"delimited_payload_filter"
}
}
}
},
"mappings": {
"movies": {
# this mapping definition sets up the metadata fields for the movies
"properties": {
"movieId": {
"type": "integer"
},
"tmdbId": {
"type": "keyword"
},
"genres": {
"type": "keyword"
},
"release_date": {
"type": "date",
"format": "year"
},
"@model": {
# this mapping definition sets up the fields for movie factor vectors of our model
"properties": {
"factor": {
"type": "binary",
"doc_values": true
},
"version": {
"type": "keyword"
},
"timestamp": {
"type": "date"
}
}
}
}}
}}
查询
You can execute the script by specifying its lang as expert_scripts, and the name of the script as the script source:
1. aggregateByKey的运行机制
/**
* Aggregate the values of each key, using given combine functions and a neutral "zero value".
* This function can return a different result type
spark-sql是Spark bin目录下的一个可执行脚本，它的目的是通过这个脚本执行Hive的命令，即原来通过
hive>输入的指令可以通过spark-sql>输入的指令来完成。
spark-sql可以使用内置的Hive metadata-store，也可以使用已经独立安装的Hive的metadata store
关于Hive build into Spark
// Max value in Array
var arr = [1,2,3,5,3,2];Math.max.apply(null, arr); // 5
// Max value in Jaon Array
var arr = [{"x":"8/11/2009","y":0.026572007},{"x"
在使用XMlhttpRequest对象发送请求和响应之前，必须首先使用javaScript对象创建一个XMLHttpRquest对象。
var xmlhttp；
function getXMLHttpRequest(){
if(window.ActiveXObject){
xmlhttp:new ActiveXObject("Microsoft.XMLHTTP
