Skip to content
Merged
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,10 @@
package com.linkedin.avroutil1.builder;

import com.linkedin.avroutil1.compatibility.AvroCodecUtil;
import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper;
import com.linkedin.avroutil1.compatibility.AvroRecordUtil;
import com.linkedin.avroutil1.compatibility.CustomDecoder;
import com.linkedin.avroutil1.compatibility.RandomRecordGenerator;
import com.linkedin.avroutil1.compatibility.RecordGenerationConfig;
import com.linkedin.avroutil1.compatibility.StringConverterUtil;

import java.io.ByteArrayOutputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
Expand All @@ -33,13 +29,9 @@
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

import com.linkedin.avroutil1.compatibility.backports.SpecificRecordBaseExt;
import noutf8.TestCollections;
import org.apache.avro.AvroRuntimeException;
import org.apache.avro.generic.IndexedRecord;
import org.apache.avro.io.Decoder;
import org.apache.avro.io.Encoder;
import org.apache.avro.util.Utf8;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
Expand Down Expand Up @@ -2048,36 +2040,6 @@ public void testNoUtf8Encoding() throws IOException {
Assert.assertTrue(instance.arOfMap.get(0).containsValue(strValue));
}

@DataProvider
private Object[][] customDecodeDataProvider() {
return new Object[][]{
{vs19.MoneyRange.class},
{vs110.MoneyRange.class},
{vs111.MoneyRange.class}
};
}

@Test(dataProvider = "customDecodeDataProvider")
public void testCustomDecode(Class<? extends SpecificRecordBaseExt> specificRecordClass) throws Exception {
RandomRecordGenerator generator = new RandomRecordGenerator();
SpecificRecordBaseExt instance = generator.randomSpecific(specificRecordClass);

ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
Encoder encoder = AvroCompatibilityHelper.newBinaryEncoder(outputStream);
Method customEncodeMethod = instance.getClass().getMethod("customEncode", Encoder.class);
customEncodeMethod.invoke(instance, encoder);
encoder.flush();

byte[] data = outputStream.toByteArray();
Decoder binaryDecoder = AvroCompatibilityHelper.newBinaryDecoder(data);
CustomDecoder decoder =
(CustomDecoder) AvroCompatibilityHelper.newCachedResolvingDecoder(
instance.getSchema(), instance.getSchema(), binaryDecoder);
SpecificRecordBaseExt decodedInstance = specificRecordClass.getDeclaredConstructor().newInstance();
decodedInstance.customDecode(decoder);
Assert.assertEquals(instance, decodedInstance);
}

@BeforeClass
public void setup() {
System.setProperty("org.apache.avro.specific.use_custom_coders", "true");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ protected JavaFile generateSpecificRecord(AvroRecordSchema recordSchema, Specifi
classBuilder.addSuperinterface(SpecificRecordGeneratorUtil.CLASSNAME_SPECIFIC_RECORD);

// extends
classBuilder.superclass(SpecificRecordGeneratorUtil.CLASSNAME_SPECIFIC_RECORD_BASE_EXT);
classBuilder.superclass(SpecificRecordGeneratorUtil.CLASSNAME_SPECIFIC_RECORD_BASE);

//add class-level doc from schema doc
//file-level (top of file) comment is added to the file object later
Expand Down Expand Up @@ -519,27 +519,9 @@ protected JavaFile generateSpecificRecord(AvroRecordSchema recordSchema, Specifi
.addParameter(SpecificRecordGeneratorUtil.CLASSNAME_RESOLVING_DECODER, "in")
.addException(IOException.class)
.addModifiers(Modifier.PUBLIC);
addCustomDecodeMethod(customDecodeBuilder, recordSchema, config, classBuilder, sizeValCounter, false);
addCustomDecodeMethod(customDecodeBuilder, recordSchema, config, classBuilder, sizeValCounter);
classBuilder.addMethod(customDecodeBuilder.build());

//customDecode with CustomDecoder
MethodSpec.Builder methodBuilder = MethodSpec
.methodBuilder("customDecode")
.addParameter(SpecificRecordGeneratorUtil.CLASSNAME_CUSTOM_DECODER, "in")
.addException(IOException.class)
.addModifiers(Modifier.PUBLIC)
.addAnnotation(Override.class);
addCustomDecodeMethod(methodBuilder, recordSchema, config, classBuilder, sizeValCounter, true);
classBuilder.addMethod(methodBuilder.build());

MethodSpec.Builder isCustomDecodingEnabledMethod = MethodSpec
.methodBuilder("isCustomDecodingEnabled")
.addModifiers(Modifier.PUBLIC)
.returns(TypeName.BOOLEAN)
.addAnnotation(Override.class)
.addCode("return hasCustomCoders();");
classBuilder.addMethod(isCustomDecodingEnabledMethod.build());

// Builder
TypeSpec.Builder recordBuilder = TypeSpec.classBuilder("Builder");
recordBuilder.addModifiers(Modifier.PUBLIC, Modifier.STATIC);
Expand Down Expand Up @@ -963,13 +945,8 @@ private String getMethodNameForFieldWithPrefix(String prefix, String fieldName)
}

private void addCustomDecodeMethod(MethodSpec.Builder customDecodeBuilder, AvroRecordSchema recordSchema,
SpecificRecordGenerationConfig config, TypeSpec.Builder classBuilder, Counter sizeValCounter, boolean isCustomDecoder) {
SpecificRecordGenerationConfig config, TypeSpec.Builder classBuilder, Counter sizeValCounter) {
int blockSize = 25, fieldCounter = 0, chunkCounter = 0;

// Decoder class name.
ClassName decoderClassName = isCustomDecoder ? SpecificRecordGeneratorUtil.CLASSNAME_CUSTOM_DECODER :
SpecificRecordGeneratorUtil.CLASSNAME_RESOLVING_DECODER;

// reset var counter
sizeValCounter.reset();
customDecodeBuilder.addStatement(
Expand All @@ -982,7 +959,7 @@ private void addCustomDecodeMethod(MethodSpec.Builder customDecodeBuilder, AvroR
customDecodeBuilder.addStatement(chunkMethodName + "(in)");
// create new method
MethodSpec.Builder customDecodeChunkMethod = MethodSpec.methodBuilder(chunkMethodName)
.addParameter(decoderClassName, "in")
.addParameter(SpecificRecordGeneratorUtil.CLASSNAME_RESOLVING_DECODER, "in")
.addException(IOException.class)
.addModifiers(Modifier.PUBLIC);
for (; fieldCounter < Math.min(blockSize * chunkCounter + blockSize, recordSchema.getFields().size());
Expand Down Expand Up @@ -1012,7 +989,7 @@ private void addCustomDecodeMethod(MethodSpec.Builder customDecodeBuilder, AvroR
customDecodeBuilder.addStatement(chunkMethodName + "(in, fieldOrder)");
// create new method
MethodSpec.Builder customDecodeChunkMethod = MethodSpec.methodBuilder(chunkMethodName)
.addParameter(decoderClassName, "in")
.addParameter(SpecificRecordGeneratorUtil.CLASSNAME_RESOLVING_DECODER, "in")
.addParameter(ArrayTypeName.of(SpecificRecordGeneratorUtil.CLASSNAME_SCHEMA_FIELD), "fieldOrder")
.addException(IOException.class)
.addModifiers(Modifier.PUBLIC);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,9 @@ public class SpecificRecordGeneratorUtil {
public static final ClassName CLASSNAME_SPECIFIC_DATA = ClassName.get("org.apache.avro.specific", "SpecificData");
public static final ClassName CLASSNAME_SPECIFIC_RECORD = ClassName.get("org.apache.avro.specific", "SpecificRecord");
public static final ClassName CLASSNAME_SPECIFIC_RECORD_BASE = ClassName.get("org.apache.avro.specific", "SpecificRecordBase");
public static final ClassName CLASSNAME_SPECIFIC_RECORD_BASE_EXT = ClassName.get("com.linkedin.avroutil1.compatibility.backports", "SpecificRecordBaseExt");
public static final ClassName CLASSNAME_SPECIFIC_DATUM_READER = ClassName.get("org.apache.avro.specific", "SpecificDatumReader");
public static final ClassName CLASSNAME_SPECIFIC_DATUM_WRITER = ClassName.get("org.apache.avro.specific", "SpecificDatumWriter");
public static final ClassName CLASSNAME_ENCODER = ClassName.get("org.apache.avro.io", "Encoder");
public static final ClassName CLASSNAME_CUSTOM_DECODER = ClassName.get("com.linkedin.avroutil1.compatibility", "CustomDecoder");
public static final ClassName CLASSNAME_RESOLVING_DECODER = ClassName.get("org.apache.avro.io", "ResolvingDecoder");
public static final ClassName CLASSNAME_DATUM_READER = ClassName.get("org.apache.avro.io", "DatumReader");
public static final ClassName CLASSNAME_DATUM_WRITER = ClassName.get("org.apache.avro.io", "DatumWriter");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@
/**
* Interface that enables custom decoding of
* {@link com.linkedin.avroutil1.compatibility.backports.SpecificRecordBaseExt} instances.
* @deprecated DO NOT USE. This class was added to support long to int type demotion and is no longer needed.
*/
@Deprecated
public interface CustomDecoder {

/** Returns the actual order in which the reader's fields will be
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@

/**
* Extension of {@link SpecificRecordBase} that allows for custom decoding using a custom decoder.
* @deprecated DO NOT USE. This class was added to support long to int type demotion and is no longer needed.
*/
@Deprecated
public abstract class SpecificRecordBaseExt extends SpecificRecordBase {

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,6 @@
import com.linkedin.avroutil1.compatibility.SkipDecoder;
import com.linkedin.avroutil1.compatibility.StringRepresentation;
import com.linkedin.avroutil1.compatibility.avro110.backports.Avro110DefaultValuesCache;
import com.linkedin.avroutil1.compatibility.avro110.backports.GenericDatumReaderExt;
import com.linkedin.avroutil1.compatibility.avro110.backports.SpecificDatumReaderExt;
import com.linkedin.avroutil1.compatibility.avro110.codec.AliasAwareSpecificDatumReader;
import com.linkedin.avroutil1.compatibility.avro110.codec.BoundedMemoryDecoder;
import com.linkedin.avroutil1.compatibility.avro110.codec.CachedResolvingDecoder;
Expand All @@ -40,6 +38,7 @@
import org.apache.avro.Schema;
import org.apache.avro.SchemaNormalization;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.io.Avro110BinaryDecoderAccessUtil;
import org.apache.avro.io.BinaryDecoder;
Expand Down Expand Up @@ -240,7 +239,7 @@ public DatumWriter<?> newGenericDatumWriter(Schema writer, GenericData genericDa

@Override
public DatumReader<?> newGenericDatumReader(Schema writer, Schema reader, GenericData genericData) {
return new GenericDatumReaderExt<>(writer, reader, genericData);
return new GenericDatumReader<>(writer, reader, genericData);
}

@Override
Expand All @@ -250,7 +249,7 @@ public DatumWriter<?> newSpecificDatumWriter(Schema writer, SpecificData specifi

@Override
public DatumReader<?> newSpecificDatumReader(Schema writer, Schema reader, SpecificData specificData) {
return new SpecificDatumReaderExt<>(writer, reader, specificData);
return new SpecificDatumReader<>(writer, reader, specificData);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@
* a specified {@link GenericData} instance under avro 1.10
*
* @param <T>
* @deprecated DO NOT USE. This class was added to support long to int type demotion and is no longer needed.
*/
@Deprecated
public class GenericDatumReaderExt<T> extends GenericDatumReader<T> {

public GenericDatumReaderExt(Schema writer, Schema reader, GenericData genericData) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@
* a specified {@link SpecificData} instance under avro 1.9.
*
* @param <T>
* @deprecated DO NOT USE. This class was added to support long to int type demotion and is no longer needed.
*/
@Deprecated
public class SpecificDatumReaderExt<T> extends SpecificDatumReader<T> {

public SpecificDatumReaderExt(Schema writer, Schema reader, SpecificData specificData) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@

package com.linkedin.avroutil1.compatibility.avro110.codec;

import com.linkedin.avroutil1.compatibility.avro110.backports.SpecificDatumReaderExt;
import org.apache.avro.Schema;
import org.apache.avro.specific.SpecificData;
import org.apache.avro.specific.SpecificDatumReader;
Expand All @@ -21,7 +20,7 @@
*
* @param <T>
*/
public class AliasAwareSpecificDatumReader<T> extends SpecificDatumReaderExt<T> {
public class AliasAwareSpecificDatumReader<T> extends SpecificDatumReader<T> {

public AliasAwareSpecificDatumReader() {
this(null, null);
Expand All @@ -41,12 +40,10 @@ public AliasAwareSpecificDatumReader(Schema writer, Schema reader) {
}

public AliasAwareSpecificDatumReader(Schema writer, Schema reader, SpecificData data) {
super(null, null, data);
throw new UnsupportedOperationException("providing custom SpecificData not supported (yet?)");
}

protected AliasAwareSpecificDatumReader(SpecificData data) {
super(null, null, data);
throw new UnsupportedOperationException("providing custom SpecificData not supported (yet?)");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.util.Utf8;

public class ResolvingDecoder extends ValidatingDecoder implements CustomDecoder {
public class ResolvingDecoder extends ValidatingDecoder implements CustomDecoder {
private Decoder backup;

ResolvingDecoder(Schema writer, Schema reader, Decoder in) throws IOException {
Expand Down Expand Up @@ -70,24 +70,6 @@ public final void drain() throws IOException {
this.parser.processImplicitActions();
}

@Override
public int readInt() throws IOException {
Symbol actual = parser.advance(Symbol.INT);

if (actual == Symbol.INT) {
return in.readInt();
} else if (actual == Symbol.IntLongAdjustAction.INSTANCE) {
long value = in.readLong();
if (value < Integer.MIN_VALUE || value > Integer.MAX_VALUE) {
throw new AvroTypeException(value + " cannot be represented as int");
}

return (int) value;
}

throw new AvroTypeException("Expected int but found " + actual);
}

public long readLong() throws IOException {
Symbol actual = this.parser.advance(Symbol.LONG);
if (actual == Symbol.INT) {
Expand Down Expand Up @@ -244,10 +226,6 @@ public Symbol doAction(Symbol input, Symbol top) throws IOException {
this.backup = this.in;
this.in = DecoderFactory.get().binaryDecoder(dsa.contents, (BinaryDecoder)null);
} else {
if (top == Symbol.IntLongAdjustAction.INSTANCE) {
return top;
}

if (top != Symbol.DEFAULT_END_ACTION) {
throw new AvroTypeException("Unknown action: " + top);
}
Expand Down
Loading