Skip to content

Commit

Permalink
Merge pull request jprante#43 from grmblfrz/7.10
Browse files Browse the repository at this point in the history 
changes for elasticsearch 7.10.x
  • Loading branch information
@jprante
jprante committed Jan 27, 2021
2 parents e513b02 + bdac066 commit b4732a1
Show file tree
Hide file tree
Showing 7 changed files with 268 additions and 188 deletions.
6 changes: 3 additions & 3 deletions gradle.properties
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
group = org.xbib.elasticsearch
name = elasticsearch-plugin-bundle
version = 7.8.1.0
version = 7.10.1.0

elasticsearch.version = 7.8.1
lucene.version = 8.5.1
elasticsearch.version = 7.10.1
lucene.version = 8.7.0

icu4j.version = 67.1
standardnumber.version = 1.0.1
Expand Down
3 changes: 2 additions & 1 deletion ...main/java/org/xbib/elasticsearch/plugin/bundle/common/decompound/fst/FstDecompounder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@ public class FstDecompounder {

public FstDecompounder(InputStream inputStream, List<String> glue) throws IOException {
try {
this.surfaceForms = new FST<>(new InputStreamDataInput(inputStream), NoOutputs.getSingleton());
InputStreamDataInput input = new InputStreamDataInput(inputStream);
this.surfaceForms = new FST<>(input, input, NoOutputs.getSingleton());
// set up glue morphemes
this.glueMorphemes = createGlueMorphemes(glue != null && glue.size() > 0 ? glue :morphemes);
} finally {
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/xbib/elasticsearch/plugin/bundle/common/fst/FstCompiler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ public void compile(InputStream inputStream, OutputStream outputStream) throws I
}
final FST<Object> fst = builder.finish();
try (OutputStreamDataOutput out = new OutputStreamDataOutput(outputStream)) {
fst.save(out);
fst.save(out, out);
}
}
}
143 changes: 77 additions & 66 deletions ...ava/org/xbib/elasticsearch/plugin/bundle/index/mapper/icu/IcuCollationKeyFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@
import com.ibm.icu.text.Collator;
import com.ibm.icu.text.RawCollationKey;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.DocValuesFieldExistsQuery;
import org.apache.lucene.search.MultiTermQuery;
Expand All @@ -20,29 +20,35 @@
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
import org.elasticsearch.index.mapper.KeywordFieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.StringFieldType;
import org.elasticsearch.index.mapper.TextSearchInfo;
import org.elasticsearch.index.mapper.TypeParsers;
import org.elasticsearch.index.mapper.ValueFetcher;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
import org.elasticsearch.search.lookup.SearchLookup;
import org.xbib.elasticsearch.plugin.bundle.index.analysis.icu.IcuCollationKeyAnalyzerProvider;
import org.xbib.elasticsearch.plugin.bundle.index.analysis.icu.IndexableBinaryStringTools;

import java.io.IOException;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.BiFunction;
import java.util.function.LongSupplier;
import java.util.function.Supplier;

/**
* ICU collation key field mapper.
Expand All @@ -51,7 +57,7 @@ public class IcuCollationKeyFieldMapper extends FieldMapper {

public static final String CONTENT_TYPE = "icu_collation_key";

public static final MappedFieldType FIELD_TYPE = new CollationFieldType();
public static final FieldType FIELD_TYPE = new FieldType();

public static class Defaults {

Expand All @@ -65,39 +71,43 @@ public static class Defaults {

public static final class CollationFieldType extends StringFieldType {

private Collator collator = null;
private final Collator collator;

public CollationFieldType() {
public CollationFieldType(String name, boolean isSearchable, boolean hasDocValues, Collator collator, Map<String, String> meta) {
super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
this.collator = collator;
}

protected CollationFieldType(CollationFieldType ref) {
super(ref);
this.collator = ref.collator;
public CollationFieldType(String name, Collator collator) {
this(name, true, true, collator, Collections.emptyMap());
}

public CollationFieldType clone() {
return new CollationFieldType(this);
@Override
public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
throw new UnsupportedOperationException();
}

@Override
public boolean equals(Object o) {
return super.equals(o) && Objects.equals(collator, ((CollationFieldType) o).collator);
public int hashCode() {
return 31 * super.hashCode() + Objects.hashCode(collator);
}

@Override
public void checkCompatibility(MappedFieldType otherFT, List<String> conflicts) {
super.checkCompatibility(otherFT,conflicts);
CollationFieldType other = (CollationFieldType) otherFT;
if (!Objects.equals(collator, other.collator)) {
conflicts.add("mapper [" + name() + "] has different [collator]");
public boolean equals(Object o)
{
if (this == o)
{
return true;
}
if (o == null || getClass() != o.getClass())
{
return false;
}
}

@Override
public int hashCode() {
return 31 * super.hashCode() + Objects.hashCode(collator);
CollationFieldType that = (CollationFieldType) o;

return collator != null ? collator.equals(that.collator) : that.collator == null;
}

@Override
Expand All @@ -109,11 +119,6 @@ public Collator collator() {
return collator;
}

public void setCollator(Collator collator) {
checkIfFrozen();
this.collator = collator.isFrozen() ? collator : collator.freeze();
}

@Override
public Query existsQuery(QueryShardContext context) {
if (hasDocValues()) {
Expand All @@ -124,9 +129,9 @@ public Query existsQuery(QueryShardContext context) {
}

@Override
public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) {
public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier<SearchLookup> searchLookup) {
failIfNoDocValues();
return new DocValuesIndexFieldData.Builder();
return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES);
}

@Override
Expand All @@ -152,12 +157,12 @@ public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int
}

@Override
public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) {
public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, QueryShardContext context) {
throw new UnsupportedOperationException();
}

@Override
public Query regexpQuery(String value, int flags, int maxDeterminizedStates,
public Query regexpQuery(String value, int syntaxFlags, int matchFlags, int maxDeterminizedStates,
MultiTermQuery.RewriteMethod method, QueryShardContext context) {
throw new UnsupportedOperationException();
}
Expand Down Expand Up @@ -216,20 +221,16 @@ public BytesRef parseBytesRef(String value) {
};
}

public static class Builder extends FieldMapper.Builder<Builder, IcuCollationKeyFieldMapper> {
public static class Builder extends FieldMapper.Builder<Builder> {
private Settings.Builder settingsBuilder;
protected String nullValue;

public Builder(String name) {
super(name, FIELD_TYPE, FIELD_TYPE);
super(name, FIELD_TYPE);
builder = this;
this.settingsBuilder = Settings.builder();
}

@Override
public CollationFieldType fieldType() {
return (CollationFieldType) super.fieldType();
}

@Override
public Builder indexOptions(IndexOptions indexOptions) {
if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) > 0) {
Expand Down Expand Up @@ -329,23 +330,27 @@ public Builder numeric(final boolean numeric) {
return this;
}

public Builder nullValue(String nullValue) {
this.nullValue = nullValue;
return this;
}

public Collator buildCollator() {
return IcuCollationKeyAnalyzerProvider.createCollator(settingsBuilder.build());
}

@Override
public IcuCollationKeyFieldMapper build(BuilderContext context) {
final Collator collator = buildCollator();
fieldType().setCollator(collator);
setupFieldType(context);
return new IcuCollationKeyFieldMapper(name, fieldType, defaultFieldType, context.indexSettings(),
multiFieldsBuilder.build(this, context), copyTo, settingsBuilder.build(), collator);
CollationFieldType ft = new CollationFieldType(buildFullName(context), indexed, hasDocValues, collator, meta);
return new IcuCollationKeyFieldMapper(name, fieldType, ft,
multiFieldsBuilder.build(this, context), copyTo, settingsBuilder.build(), collator, nullValue);
}
}

public static class TypeParser implements Mapper.TypeParser {
@Override
public Mapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext)
public Mapper.Builder<?> parse(String name, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
Builder builder = new Builder(name);
TypeParsers.parseField(builder, name, node, parserContext);
Expand Down Expand Up @@ -415,16 +420,16 @@ public static class TypeParser implements Mapper.TypeParser {

private final Settings collatorSettings;
private final Collator collator;
private final BiFunction<String, BytesRef, Field> getDVField;
private final String nullValue;

protected IcuCollationKeyFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
Settings indexSettings, MultiFields multiFields,
CopyTo copyTo, Settings collatorSettings, Collator collator) {
super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
protected IcuCollationKeyFieldMapper(String simpleName, FieldType fieldType, MappedFieldType defaultFieldType,
MultiFields multiFields,
CopyTo copyTo, Settings collatorSettings, Collator collator, String nullValue) {
super(simpleName, fieldType, defaultFieldType, multiFields, copyTo);
assert collator.isFrozen();
this.collatorSettings = collatorSettings;
this.collator = collator;
this.getDVField = SortedSetDocValuesField::new;
this.nullValue = nullValue;
}

@Override
Expand All @@ -438,10 +443,11 @@ protected String contentType() {
}

@Override
protected void doMerge(Mapper mergeWith) {
super.doMerge(mergeWith);
List<String> conflicts = new ArrayList<>();
IcuCollationKeyFieldMapper icuMergeWith = (IcuCollationKeyFieldMapper) mergeWith;
protected void mergeOptions(FieldMapper other, List<String> conflicts) {
IcuCollationKeyFieldMapper icuMergeWith = (IcuCollationKeyFieldMapper) other;
if (!Objects.equals(collator, icuMergeWith.collator)) {
conflicts.add("mapper [" + name() + "] has different [collator]");
}
if (!Objects.equals(collatorSettings.get("rules"), icuMergeWith.collatorSettings.get("rules"))) {
conflicts.add("Cannot update rules setting for [" + CONTENT_TYPE + "]");
}
Expand Down Expand Up @@ -472,17 +478,20 @@ protected void doMerge(Mapper mergeWith) {
if (collatorSettings.getAsBoolean("numeric", true) != icuMergeWith.collatorSettings.getAsBoolean("numeric", true)) {
conflicts.add("Cannot update numeric setting for [" + CONTENT_TYPE + "]");
}
if (!conflicts.isEmpty()) {
throw new IllegalArgumentException("Can't merge because of conflicts: " + conflicts);
}
}


@Override
protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
super.doXContentBody(builder, includeDefaults, params);
if (includeDefaults || fieldType().nullValue() != null) {
builder.field("null_value", fieldType().nullValue());
if (fieldType.indexOptions() != IndexOptions.NONE && (includeDefaults || fieldType.indexOptions() != IndexOptions.DOCS)) {
builder.field("index_options", indexOptionToString(fieldType.indexOptions()));
}
if (nullValue != null) {
builder.field("null_value", nullValue);
}
if (includeDefaults || fieldType.omitNorms() != KeywordFieldMapper.Defaults.FIELD_TYPE.omitNorms()) {
builder.field("norms", fieldType.omitNorms() == false);
}
if (includeDefaults) {
builder.field("rules", collatorSettings.get("rules"));
Expand Down Expand Up @@ -517,14 +526,14 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
}

@Override
protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
protected void parseCreateField(ParseContext context) throws IOException {
final String value;
if (context.externalValueSet()) {
value = context.externalValue().toString();
} else {
XContentParser parser = context.parser();
if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
value = fieldType().nullValueAsString();
value = nullValue;
} else {
value = parser.textOrNull();
}
Expand All @@ -534,12 +543,14 @@ protected void parseCreateField(ParseContext context, List<IndexableField> field
}
RawCollationKey key = collator.getRawCollationKey(value, null);
final BytesRef binaryValue = new BytesRef(key.bytes, 0, key.size);
if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
Field field = new Field(fieldType().name(), binaryValue, fieldType());
fields.add(field);
if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
Field field = new Field(mappedFieldType.name(), binaryValue, fieldType);
context.doc().add(field);
}
if (fieldType().hasDocValues()) {
fields.add(getDVField.apply(fieldType().name(), binaryValue));
context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue));
} else if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
createFieldNamesField(context);
}
}
}
Loading

0 comments on commit b4732a1

Please sign in to comment.