implemented plugin and unit tests
This commit is contained in:
parent
0df28bc0b9
commit
b4e6209f9a
3 changed files with 113 additions and 40 deletions
|
@ -17,59 +17,77 @@
|
||||||
|
|
||||||
package usi.vaa.elasticsearch.plugin.ingest.lookup;
|
package usi.vaa.elasticsearch.plugin.ingest.lookup;
|
||||||
|
|
||||||
import org.elasticsearch.common.Strings;
|
|
||||||
import org.elasticsearch.ingest.AbstractProcessor;
|
import org.elasticsearch.ingest.AbstractProcessor;
|
||||||
import org.elasticsearch.ingest.IngestDocument;
|
import org.elasticsearch.ingest.IngestDocument;
|
||||||
import org.elasticsearch.ingest.Processor;
|
import org.elasticsearch.ingest.Processor;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.util.Collections;
|
||||||
|
import java.util.HashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.regex.MatchResult;
|
||||||
|
import java.util.regex.Matcher;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
import static org.elasticsearch.ingest.ConfigurationUtils.readMap;
|
import static org.elasticsearch.ingest.ConfigurationUtils.readMap;
|
||||||
import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
|
import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
|
||||||
|
|
||||||
public class LookupProcessor extends AbstractProcessor {
|
public class LookupProcessor extends AbstractProcessor {
|
||||||
|
public static final String FIELD_PROPERTY = "field";
|
||||||
|
public static final String LOOKUP_MAP_PROPERTY = "lookup-map";
|
||||||
public static final String TYPE = "lookup";
|
public static final String TYPE = "lookup";
|
||||||
|
private static final Pattern PATTERN = Pattern.compile("([^\\s\\p{Z}]+)([\\s\\p{Z}]+|$)");
|
||||||
|
private final String field;
|
||||||
|
private final Map<String, Object> lookupMap;
|
||||||
|
|
||||||
private final String field = "";
|
public LookupProcessor(String tag, String description, String field, Map<String, String> lookupMap) {
|
||||||
|
|
||||||
|
|
||||||
// TODO this signature and method might be incomplete - change as needed
|
|
||||||
public LookupProcessor(String tag, String description, String field,
|
|
||||||
Map<String,String> lookupMap) throws IOException {
|
|
||||||
super(tag, description);
|
super(tag, description);
|
||||||
|
this.field = field;
|
||||||
|
this.lookupMap = new HashMap<>(lookupMap);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String replaceWord(final MatchResult toReplace) {
|
||||||
|
if (toReplace.groupCount() != 2) {
|
||||||
|
throw new IllegalArgumentException("Captured groups should be coherent with PATTERN regex");
|
||||||
|
}
|
||||||
|
|
||||||
|
final String word = toReplace.group(1);
|
||||||
|
final String spacesAfterWord = toReplace.group(2);
|
||||||
|
|
||||||
|
return lookupMap.getOrDefault(word, word) + spacesAfterWord;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public IngestDocument execute(IngestDocument ingestDocument) throws Exception{
|
public IngestDocument execute(IngestDocument ingestDocument) {
|
||||||
String originalContent = ingestDocument.getFieldValue(field, String.class);
|
final String originalContent = ingestDocument.getFieldValue(field, String.class);
|
||||||
ingestDocument.setFieldValue(field, "Hello world");
|
final Matcher matcher = PATTERN.matcher(originalContent);
|
||||||
return null;
|
final String replacedContent = matcher.replaceAll(this::replaceWord);
|
||||||
|
ingestDocument.setFieldValue(field, replacedContent);
|
||||||
|
return ingestDocument;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getType() {
|
public String getType() {
|
||||||
// TODO your job!
|
return TYPE;
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
String getField() {
|
String getField() {
|
||||||
// TODO your job!
|
return this.field;
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Map<String, Object> getLookupMap() {
|
||||||
|
return Collections.unmodifiableMap(lookupMap);
|
||||||
|
}
|
||||||
|
|
||||||
public static final class Factory implements Processor.Factory {
|
public static final class Factory implements Processor.Factory {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public LookupProcessor create(Map<String, Processor.Factory> factories, String tag,
|
public LookupProcessor create(final Map<String, Processor.Factory> factories,
|
||||||
String description, Map<String, Object> config)
|
final String tag,
|
||||||
throws Exception {
|
final String description,
|
||||||
String field = readStringProperty(TYPE, tag, config, "field");
|
final Map<String, Object> config) {
|
||||||
|
final String field = readStringProperty(TYPE, tag, config, FIELD_PROPERTY);
|
||||||
// TODO your job! Also initialize the lookup processor properly if needed, after you changed the constructor
|
final Map<String, String> lookupMap = readMap(TYPE, tag, config, LOOKUP_MAP_PROPERTY);
|
||||||
return null;
|
return new LookupProcessor(tag, description, field, lookupMap);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -24,25 +24,47 @@ import org.junit.BeforeClass;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
import static org.hamcrest.Matchers.equalTo;
|
import static usi.vaa.elasticsearch.plugin.ingest.lookup.LookupProcessor.*;
|
||||||
|
|
||||||
public class LookupProcessorFactoryTests extends ESTestCase {
|
public class LookupProcessorFactoryTests extends ESTestCase {
|
||||||
|
private static String tag;
|
||||||
|
private static String description;
|
||||||
|
private static LookupProcessor.Factory factory;
|
||||||
|
|
||||||
@BeforeClass
|
@BeforeClass
|
||||||
public static void defaultSettings() {
|
public static void defaultSettings() {
|
||||||
// TODO your job!
|
tag = "lookup-test";
|
||||||
|
description = "lookup pipeline test";
|
||||||
|
factory = new LookupProcessor.Factory();
|
||||||
}
|
}
|
||||||
|
|
||||||
private Map<String, Object> getDefaultConfig() {
|
private LookupProcessor callFactory(final Map<String, Object> config) {
|
||||||
// TODO your job!
|
// if the `config` map is not cloned here java.lang.UnsupportedOperationException is thrown
|
||||||
|
// for some reflective magic related reason, therefore do not remove
|
||||||
|
return factory.create(null, tag, description, new HashMap<>(config));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testDefaultConfiguration() throws Exception{
|
public void testDefaultConfiguration() {
|
||||||
// TODO your job!
|
final String testField = "theTestField";
|
||||||
|
final Map<String, String> testMap = Map.of(
|
||||||
|
"C001", "tyre",
|
||||||
|
"C010", "front wing",
|
||||||
|
"C100", "damper"
|
||||||
|
);
|
||||||
|
|
||||||
|
final Map<String, Object> config = Map.of(
|
||||||
|
FIELD_PROPERTY, testField,
|
||||||
|
LOOKUP_MAP_PROPERTY, testMap
|
||||||
|
);
|
||||||
|
final LookupProcessor processor = callFactory(config);
|
||||||
|
|
||||||
|
assertEquals(tag, processor.getTag());
|
||||||
|
assertEquals(description, processor.getDescription());
|
||||||
|
assertEquals(testMap, processor.getLookupMap());
|
||||||
|
assertEquals(testField, processor.getField());
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testInvalidConfiguration() throws Exception{
|
public void testInvalidConfiguration() {
|
||||||
// TODO your job!
|
expectThrows(ElasticsearchParseException.class, () -> callFactory(Map.of("invalid", "config")));
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -22,25 +22,58 @@ import org.elasticsearch.ingest.IngestDocument;
|
||||||
import org.elasticsearch.ingest.RandomDocumentPicks;
|
import org.elasticsearch.ingest.RandomDocumentPicks;
|
||||||
import org.elasticsearch.test.ESTestCase;
|
import org.elasticsearch.test.ESTestCase;
|
||||||
import org.junit.BeforeClass;
|
import org.junit.BeforeClass;
|
||||||
|
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
import static org.hamcrest.Matchers.equalTo;
|
import static usi.vaa.elasticsearch.plugin.ingest.lookup.LookupProcessor.FIELD_PROPERTY;
|
||||||
|
import static usi.vaa.elasticsearch.plugin.ingest.lookup.LookupProcessor.LOOKUP_MAP_PROPERTY;
|
||||||
|
|
||||||
public class LookupProcessorTests extends ESTestCase {
|
public class LookupProcessorTests extends ESTestCase {
|
||||||
|
private static final String DOC_FIELD = "field_to_replace";
|
||||||
|
private static LookupProcessor processor;
|
||||||
|
|
||||||
@BeforeClass
|
@BeforeClass
|
||||||
public static void defaultSettings() {
|
public static void defaultSettings() {
|
||||||
// TODO your job!
|
processor = new LookupProcessor.Factory().create(
|
||||||
|
null,
|
||||||
|
"test-lookup",
|
||||||
|
"Test lookup processor pipeline",
|
||||||
|
new HashMap<>(Map.of(
|
||||||
|
FIELD_PROPERTY, DOC_FIELD,
|
||||||
|
LOOKUP_MAP_PROPERTY, Map.of(
|
||||||
|
"C001", "tyre",
|
||||||
|
"C010", "front wing",
|
||||||
|
"C100", "damper"
|
||||||
|
)
|
||||||
|
))
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testSimple() throws Exception{
|
private IngestDocument getDocWithContent(final String content) {
|
||||||
// TODO your job!
|
final Map<String, Object> doc = Map.of(DOC_FIELD, content);
|
||||||
|
return RandomDocumentPicks.randomIngestDocument(random(), doc);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testEmptyField() throws Exception{
|
private void testWithContent(final String originalContent, final String expectedContent) {
|
||||||
// TODO your job!
|
final IngestDocument doc = getDocWithContent(originalContent);
|
||||||
|
processor.execute(doc);
|
||||||
|
assertEquals(expectedContent, doc.getSourceAndMetadata().get(DOC_FIELD));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void testSimple() {
|
||||||
|
final String originalContent = "Need to optimize the C001 temperature.\n" +
|
||||||
|
"C010 needs to be changed as it is damaged.\tC100 seems ok.";
|
||||||
|
final String expectedContent = "Need to optimize the tyre temperature.\n" +
|
||||||
|
"front wing needs to be changed as it is damaged.\tdamper seems ok.";
|
||||||
|
|
||||||
|
testWithContent(originalContent, expectedContent);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testEmptyField() {
|
||||||
|
final String originalContent = "";
|
||||||
|
final String expectedContent = "";
|
||||||
|
|
||||||
|
testWithContent(originalContent, expectedContent);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Reference in a new issue