diff options
| author | Matthias Andreas Benkard <code@mail.matthias.benkard.de> | 2023-12-10 09:20:48 +0100 | 
|---|---|---|
| committer | Matthias Andreas Benkard <code@mail.matthias.benkard.de> | 2023-12-10 09:22:48 +0100 | 
| commit | aa11d82fe887d7c625c8bd0e89e2947e448d8bb4 (patch) | |
| tree | 5080d40d3eb6d335916cfc8ef62403417236bcae | |
| parent | 04a5ce11203665fe1f03547bcfb6618ec0915c38 (diff) | |
Add Decoder#encode.
Implements:
 - the encoding part of the GVariant specification
 - OSTree-specific encoding instructions for static deltas
Untested.
Change-Id: Idbfd6d7e92a9cdff7d8b138d0ecfa36d4f30eee4
12 files changed, 507 insertions, 36 deletions
| diff --git a/jgvariant-core/src/main/java/eu/mulk/jgvariant/core/Decoder.java b/jgvariant-core/src/main/java/eu/mulk/jgvariant/core/Decoder.java index 4538900..a28d792 100644 --- a/jgvariant-core/src/main/java/eu/mulk/jgvariant/core/Decoder.java +++ b/jgvariant-core/src/main/java/eu/mulk/jgvariant/core/Decoder.java @@ -4,15 +4,21 @@  package eu.mulk.jgvariant.core; +import static java.lang.Math.max;  import static java.nio.ByteOrder.LITTLE_ENDIAN; +import static java.nio.charset.StandardCharsets.UTF_8;  import static java.util.Objects.requireNonNullElse;  import static java.util.stream.Collectors.toMap;  import com.google.errorprone.annotations.Immutable; + +import java.io.ByteArrayOutputStream; +import java.io.IOException;  import java.lang.reflect.InvocationTargetException;  import java.lang.reflect.RecordComponent;  import java.nio.ByteBuffer;  import java.nio.ByteOrder; +import java.nio.channels.Channels;  import java.nio.charset.Charset;  import java.text.ParseException;  import java.util.ArrayList; @@ -25,6 +31,7 @@ import java.util.Optional;  import java.util.function.Function;  import java.util.function.Predicate;  import java.util.function.UnaryOperator; +  import org.apiguardian.api.API;  import org.apiguardian.api.API.Status;  import org.jetbrains.annotations.NotNull; @@ -80,10 +87,24 @@ public abstract class Decoder<T> {     */    public abstract @NotNull T decode(ByteBuffer byteSlice); +  /** +   * Encodes a value of type {@code T} into a {@link ByteBuffer} holding a serialized GVariant. +   * +   * @param value the value to serialize. +   * @return a {@link ByteBuffer} holding the serialized value. +   */ +  public final ByteBuffer encode(T value) { +    var byteWriter = new ByteWriter(); +    encode(value, byteWriter); +    return byteWriter.toByteBuffer(); +  } +    abstract byte alignment();    abstract @Nullable Integer fixedSize(); +  abstract void encode(T value, ByteWriter byteWriter); +    final boolean hasFixedSize() {      return fixedSize() != null;    } @@ -105,8 +126,8 @@ public abstract class Decoder<T> {     * @return a new, decorated {@link Decoder}.     * @see java.util.stream.Stream#map     */ -  public final <U> Decoder<U> map(Function<@NotNull T, @NotNull U> function) { -    return new MappingDecoder<>(function); +  public final <U> Decoder<U> map(Function<@NotNull T, @NotNull U> decodingFunction, Function<@NotNull U, @NotNull T> encodingFunction) { +    return new MappingDecoder<>(decodingFunction, encodingFunction);    }    /** @@ -116,8 +137,8 @@ public abstract class Decoder<T> {     * @return a new, decorated {@link Decoder}.     * @see java.util.stream.Stream#map     */ -  public final Decoder<T> contramap(UnaryOperator<ByteBuffer> function) { -    return new ContramappingDecoder(function); +  public final Decoder<T> contramap(UnaryOperator<ByteBuffer> decodingFunction, UnaryOperator<ByteBuffer> encodingFunction) { +    return new ContramappingDecoder(decodingFunction, encodingFunction);    }    /** @@ -335,6 +356,22 @@ public abstract class Decoder<T> {      return n < (1 << 8) ? 1 : n < (1 << 16) ? 2 : 4;    } +  private static int computeFramingOffsetSize(int elementsRelativeEnd, List<Integer> framingOffsets) { +    // Determining the framing offset size requires trial and error. +    int framingOffsetSize; +    for (framingOffsetSize = 0;; framingOffsetSize = max(1, framingOffsetSize << 1)) { +      if (elementsRelativeEnd + framingOffsetSize* framingOffsets.size() >= 1 << (8*framingOffsetSize)) { +        continue; +      } + +      if (framingOffsetSize > 4) { +        throw new IllegalArgumentException("too many framing offsets"); +      } + +      return framingOffsetSize; +    } +  } +    private static class ArrayDecoder<U> extends Decoder<List<U>> {      private final Decoder<U> elementDecoder; @@ -391,6 +428,33 @@ public abstract class Decoder<T> {        return elements;      } + +    @Override +    void encode(List<U> value, ByteWriter byteWriter) { +      if (elementDecoder.hasFixedSize()) { +        for (var element : value) { +          elementDecoder.encode(element, byteWriter); +        } +      } else { +        // Variable-width arrays are encoded with a vector of framing offsets in the end. +        ArrayList<Integer> framingOffsets = new ArrayList<>(value.size()); +        int startOffset = byteWriter.position(); +        for (var element : value) { +          elementDecoder.encode(element, byteWriter); +          var relativeEnd = byteWriter.position() - startOffset; +          framingOffsets.add(relativeEnd); + +          // Align the next element. +          byteWriter.write(new byte[align(relativeEnd, alignment()) - relativeEnd]); +        } + +        // Write the framing offsets. +        int framingOffsetSize = computeFramingOffsetSize(byteWriter.position() - startOffset, framingOffsets); +        for (var framingOffset : framingOffsets) { +          byteWriter.writeIntN(framingOffset, framingOffsetSize); +        } +      } +    }    }    private static class DictionaryDecoder<K, V> extends Decoder<Map<K, V>> { @@ -418,6 +482,11 @@ public abstract class Decoder<T> {        List<Map.Entry<K, V>> entries = entryArrayDecoder.decode(byteSlice);        return entries.stream().collect(toMap(Entry::getKey, Entry::getValue));      } + +    @Override +    void encode(Map<K, V> value, ByteWriter byteWriter) { +      entryArrayDecoder.encode(value.entrySet().stream().toList(), byteWriter); +    }    }    private static class ByteArrayDecoder extends Decoder<byte[]> { @@ -442,6 +511,11 @@ public abstract class Decoder<T> {        byteSlice.get(elements);        return elements;      } + +    @Override +    void encode(byte[] value, ByteWriter byteWriter) { +      byteWriter.write(value); +    }    }    private static class MaybeDecoder<U> extends Decoder<Optional<U>> { @@ -476,6 +550,18 @@ public abstract class Decoder<T> {          return Optional.of(elementDecoder.decode(byteSlice));        }      } + +    @Override +    void encode(Optional<U> value, ByteWriter byteWriter) { +      if (value.isEmpty()) { +        return; +      } + +      elementDecoder.encode(value.get(), byteWriter); +      if (!elementDecoder.hasFixedSize()) { +        byteWriter.write((byte) 0); +      } +    }    }    private static class StructureDecoder<U extends Record> extends Decoder<U> { @@ -523,6 +609,22 @@ public abstract class Decoder<T> {          throw new IllegalStateException(e);        }      } + +    @Override +    void encode(U value, ByteWriter byteWriter) { +      try { +        var components = recordType.getRecordComponents(); +        List<Object> componentValues = new ArrayList<>(components.length); +        for (var component : components) { +          var accessor = component.getAccessor(); +          var componentValue = accessor.invoke(value); +          componentValues.add(componentValue); +        } +        tupleDecoder.encode(componentValues.toArray(), byteWriter); +      } catch (IllegalAccessException | InvocationTargetException e) { +        throw new IllegalStateException(e); +      } +    }    }    @SuppressWarnings("Immutable") @@ -604,6 +706,33 @@ public abstract class Decoder<T> {        return objects;      } + +    @Override +    @SuppressWarnings("unchecked") +    void encode(Object[] value, ByteWriter byteWriter) { +      int startOffset = byteWriter.position(); +      ArrayList<Integer> framingOffsets = new ArrayList<>(value.length); +      for (int i = 0; i < value.length; ++i) { +        var componentDecoder = (Decoder<Object>) componentDecoders[i]; +        componentDecoder.encode(value[i], byteWriter); + +        var relativeEnd = byteWriter.position() - startOffset; + +        var fixedComponentSize = componentDecoders[i].fixedSize(); +        if (fixedComponentSize == null && i < value.length - 1) { +          framingOffsets.add(relativeEnd); +        } + +        // Align the next element. +        byteWriter.write(new byte[align(relativeEnd, alignment()) - relativeEnd]); +      } + +      // Write the framing offsets in reverse order. +      int framingOffsetSize = computeFramingOffsetSize(byteWriter.position() - startOffset, framingOffsets); +      for (int i = framingOffsets.size() - 1; i >= 0; --i) { +        byteWriter.writeIntN(framingOffsets.get(i), framingOffsetSize); +      } +    }    }    private static class DictionaryEntryDecoder<K, V> extends Decoder<Map.Entry<K, V>> { @@ -630,6 +759,11 @@ public abstract class Decoder<T> {        Object[] components = tupleDecoder.decode(byteSlice);        return Map.entry((K) components[0], (V) components[1]);      } + +    @Override +    void encode(Entry<K, V> value, ByteWriter byteWriter) { +      tupleDecoder.encode(new Object[] {value.getKey(), value.getValue()}, byteWriter); +    }    }    private static class VariantDecoder extends Decoder<Variant> { @@ -667,6 +801,13 @@ public abstract class Decoder<T> {        throw new IllegalArgumentException("variant signature not found");      } + +    @Override +    void encode(Variant value, ByteWriter byteWriter) { +      value.signature().decoder().encode(value.value(), byteWriter); +      byteWriter.write((byte) 0); +      byteWriter.write(value.signature().toString().getBytes(UTF_8)); +    }    }    private static class BooleanDecoder extends Decoder<Boolean> { @@ -685,6 +826,11 @@ public abstract class Decoder<T> {      public @NotNull Boolean decode(ByteBuffer byteSlice) {        return byteSlice.get() != 0;      } + +    @Override +    void encode(Boolean value, ByteWriter byteWriter) { +      byteWriter.write(Boolean.TRUE.equals(value) ? (byte) 1 : (byte) 0); +    }    }    private static class ByteDecoder extends Decoder<Byte> { @@ -703,6 +849,11 @@ public abstract class Decoder<T> {      public @NotNull Byte decode(ByteBuffer byteSlice) {        return byteSlice.get();      } + +    @Override +    void encode(Byte value, ByteWriter byteWriter) { +      byteWriter.write(value); +    }    }    private static class ShortDecoder extends Decoder<Short> { @@ -721,6 +872,11 @@ public abstract class Decoder<T> {      public @NotNull Short decode(ByteBuffer byteSlice) {        return byteSlice.getShort();      } + +    @Override +    void encode(Short value, ByteWriter byteWriter) { +      byteWriter.write(value); +    }    }    private static class IntegerDecoder extends Decoder<Integer> { @@ -739,6 +895,11 @@ public abstract class Decoder<T> {      public @NotNull Integer decode(ByteBuffer byteSlice) {        return byteSlice.getInt();      } + +    @Override +    void encode(Integer value, ByteWriter byteWriter) { +      byteWriter.write(value); +    }    }    private static class LongDecoder extends Decoder<Long> { @@ -757,6 +918,11 @@ public abstract class Decoder<T> {      public @NotNull Long decode(ByteBuffer byteSlice) {        return byteSlice.getLong();      } + +    @Override +    void encode(Long value, ByteWriter byteWriter) { +      byteWriter.write(value); +    }    }    private static class DoubleDecoder extends Decoder<Double> { @@ -775,6 +941,11 @@ public abstract class Decoder<T> {      public @NotNull Double decode(ByteBuffer byteSlice) {        return byteSlice.getDouble();      } + +    @Override +    void encode(Double value, ByteWriter byteWriter) { +      byteWriter.write(value); +    }    }    private static class StringDecoder extends Decoder<String> { @@ -801,15 +972,23 @@ public abstract class Decoder<T> {        byteSlice.limit(byteSlice.limit() - 1);        return charset.decode(byteSlice).toString();      } + +    @Override +    void encode(String value, ByteWriter byteWriter) { +      byteWriter.write(charset.encode(value)); +      byteWriter.write((byte) 0); +    }    }    @SuppressWarnings("Immutable")    private class MappingDecoder<U> extends Decoder<U> { -    private final Function<@NotNull T, @NotNull U> function; +    private final Function<@NotNull T, @NotNull U> decodingFunction; +    private final Function<@NotNull U, @NotNull T> encodingFunction; -    MappingDecoder(Function<@NotNull T, @NotNull U> function) { -      this.function = function; +    MappingDecoder(Function<@NotNull T, @NotNull U> decodingFunction, Function<@NotNull U, @NotNull T> encodingFunction) { +      this.decodingFunction = decodingFunction; +      this.encodingFunction = encodingFunction;      }      @Override @@ -824,17 +1003,24 @@ public abstract class Decoder<T> {      @Override      public @NotNull U decode(ByteBuffer byteSlice) { -      return function.apply(Decoder.this.decode(byteSlice)); +      return decodingFunction.apply(Decoder.this.decode(byteSlice)); +    } + +    @Override +    void encode(U value, ByteWriter byteWriter) { +      Decoder.this.encode(encodingFunction.apply(value), byteWriter);      }    }    @SuppressWarnings("Immutable")    private class ContramappingDecoder extends Decoder<T> { -    private final UnaryOperator<ByteBuffer> function; +    private final UnaryOperator<ByteBuffer> decodingFunction; +    private final UnaryOperator<ByteBuffer> encodingFunction; -    ContramappingDecoder(UnaryOperator<ByteBuffer> function) { -      this.function = function; +    ContramappingDecoder(UnaryOperator<ByteBuffer> decodingFunction, UnaryOperator<ByteBuffer> encodingFunction) { +      this.decodingFunction = decodingFunction; +      this.encodingFunction = encodingFunction;      }      @Override @@ -849,9 +1035,17 @@ public abstract class Decoder<T> {      @Override      public @NotNull T decode(ByteBuffer byteSlice) { -      var transformedBuffer = function.apply(byteSlice.asReadOnlyBuffer().order(byteSlice.order())); +      var transformedBuffer = decodingFunction.apply(byteSlice.asReadOnlyBuffer().order(byteSlice.order()));        return Decoder.this.decode(transformedBuffer);      } + +    @Override +    void encode(T value, ByteWriter byteWriter) { +      var innerByteWriter = new ByteWriter(); +      Decoder.this.encode(value, innerByteWriter); +      var transformedBuffer = encodingFunction.apply(innerByteWriter.toByteBuffer()); +      byteWriter.write(transformedBuffer); +    }    }    private class ByteOrderFixingDecoder extends Decoder<T> { @@ -878,6 +1072,13 @@ public abstract class Decoder<T> {        newByteSlice.order(byteOrder);        return Decoder.this.decode(newByteSlice);      } + +    @Override +    protected void encode(T value, ByteWriter byteWriter) { +      var newByteWriter = byteWriter.duplicate(); +      newByteWriter.order(byteOrder); +      Decoder.this.encode(value, newByteWriter); +    }    }    private static ByteBuffer slicePreservingOrder(ByteBuffer byteSlice, int index, int length) { @@ -927,5 +1128,92 @@ public abstract class Decoder<T> {        byteSlice.rewind();        return b ? thenDecoder.decode(byteSlice) : elseDecoder.decode(byteSlice);      } + +    @Override +    public void encode(U value, ByteWriter byteWriter) { +      elseDecoder.encode(value, byteWriter); +    } +  } + +  private static class ByteWriter { +    private ByteOrder byteOrder = ByteOrder.nativeOrder(); +    private final ByteArrayOutputStream outputStream; + +    ByteWriter() { +      this.outputStream = new ByteArrayOutputStream(); +    } + +    private ByteWriter(ByteArrayOutputStream outputStream) { +      this.outputStream = outputStream; +    } + +    void write(byte[] bytes) { +      outputStream.write(bytes, 0, bytes.length); +    } + +    @SuppressWarnings("java:S2095") +    void write(ByteBuffer byteBuffer) { +      var channel = Channels.newChannel(outputStream); +      try { +        channel.write(byteBuffer); +      } catch (IOException e) { +        // impossible +        throw new IllegalStateException(e); +      } +    } + +    void write(byte value) { +      outputStream.write(value); +    } + +    void write(int value) { +      write(ByteBuffer.allocate(4).order(byteOrder).putInt(value)); +    } + +    void write(long value) { +      write(ByteBuffer.allocate(8).order(byteOrder).putLong(value)); +    } + +    void write(short value) { +      write(ByteBuffer.allocate(2).order(byteOrder).putShort(value)); +    } + +    void write(double value) { +      write(ByteBuffer.allocate(8).order(byteOrder).putDouble(value)); +    } + +    private void writeIntN(int value, int byteCount) { +      var byteBuffer = ByteBuffer.allocate(byteCount).order(LITTLE_ENDIAN); +        switch (byteCount) { +          case 0 -> {} +          case 1 -> +            byteBuffer.put((byte) value); +          case 2 -> +            byteBuffer.putShort((short) value); +          case 4 -> +            byteBuffer.putInt(value); +          default -> +            throw new IllegalArgumentException("invalid byte count: %d".formatted(byteCount)); +        } +      write(byteBuffer); +    } + +    ByteWriter duplicate() { +        var duplicate = new ByteWriter(outputStream); +        duplicate.byteOrder = byteOrder; +        return duplicate; +    } + +    ByteBuffer toByteBuffer() { +      return ByteBuffer.wrap(outputStream.toByteArray()); +    } + +    void order(ByteOrder byteOrder) { +        this.byteOrder = byteOrder; +    } + +    int position() { +      return outputStream.size(); +    }    }  } diff --git a/jgvariant-core/src/test/java/eu/mulk/jgvariant/core/DecoderTest.java b/jgvariant-core/src/test/java/eu/mulk/jgvariant/core/DecoderTest.java index 6399f6e..068b051 100644 --- a/jgvariant-core/src/test/java/eu/mulk/jgvariant/core/DecoderTest.java +++ b/jgvariant-core/src/test/java/eu/mulk/jgvariant/core/DecoderTest.java @@ -517,21 +517,39 @@ class DecoderTest {    @Test    void map() {      var data = new byte[] {0x0A, 0x0B, 0x0C}; -    var decoder = Decoder.ofByteArray().map(bytes -> bytes.length); +    var decoder = +        Decoder.ofByteArray() +            .map( +                bytes -> bytes.length, +                len -> { +                  throw new UnsupportedOperationException(); +                });      assertEquals(3, decoder.decode(ByteBuffer.wrap(data)));    }    @Test    void contramap() {      var data = new byte[] {0x0A, 0x0B, 0x0C}; -    var decoder = Decoder.ofByteArray().contramap(bytes -> bytes.slice(1, 1)); +    var decoder = +        Decoder.ofByteArray() +            .contramap( +                bytes -> bytes.slice(1, 1), +                bytes -> { +                  throw new UnsupportedOperationException(); +                });      assertArrayEquals(new byte[] {0x0B}, decoder.decode(ByteBuffer.wrap(data)));    }    @Test    void predicateTrue() {      var data = new byte[] {0x00, 0x01, 0x00}; -    var innerDecoder = Decoder.ofShort().contramap(bytes -> bytes.slice(1, 2).order(bytes.order())); +    var innerDecoder = +        Decoder.ofShort() +            .contramap( +                bytes -> bytes.slice(1, 2).order(bytes.order()), +                bytes -> { +                  throw new UnsupportedOperationException(); +                });      var decoder =          Decoder.ofPredicate(              byteBuffer -> byteBuffer.get(0) == 0, @@ -543,7 +561,13 @@ class DecoderTest {    @Test    void predicateFalse() {      var data = new byte[] {0x01, 0x01, 0x00}; -    var innerDecoder = Decoder.ofShort().contramap(bytes -> bytes.slice(1, 2).order(bytes.order())); +    var innerDecoder = +        Decoder.ofShort() +            .contramap( +                bytes -> bytes.slice(1, 2).order(bytes.order()), +                bytes -> { +                  throw new UnsupportedOperationException(); +                });      var decoder =          Decoder.ofPredicate(              byteBuffer -> byteBuffer.get(0) == 0, diff --git a/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/ByteString.java b/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/ByteString.java index cfe3635..3bd8b25 100644 --- a/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/ByteString.java +++ b/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/ByteString.java @@ -20,7 +20,8 @@ import org.jetbrains.annotations.Nullable;   */  public record ByteString(byte[] bytes) { -  private static final Decoder<ByteString> DECODER = Decoder.ofByteArray().map(ByteString::new); +  private static final Decoder<ByteString> DECODER = +      Decoder.ofByteArray().map(ByteString::new, ByteString::bytes);    /**     * Returns a decoder for a {@code byte[]} that wraps the result in {@link ByteString}. diff --git a/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/Checksum.java b/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/Checksum.java index 261e2be..829664e 100644 --- a/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/Checksum.java +++ b/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/Checksum.java @@ -17,7 +17,8 @@ public record Checksum(ByteString byteString) {    private static final int SIZE = 32; -  private static final Decoder<Checksum> DECODER = ByteString.decoder().map(Checksum::new); +  private static final Decoder<Checksum> DECODER = +      ByteString.decoder().map(Checksum::new, Checksum::byteString);    public Checksum {      if (byteString.size() == 0) { diff --git a/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/DeltaFallback.java b/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/DeltaFallback.java index 57c8fc5..08e0b8c 100644 --- a/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/DeltaFallback.java +++ b/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/DeltaFallback.java @@ -25,7 +25,7 @@ public record DeltaFallback(    private static final Decoder<DeltaFallback> DECODER =        Decoder.ofStructure(            DeltaFallback.class, -          Decoder.ofByte().map(ObjectType::valueOf), +          Decoder.ofByte().map(ObjectType::valueOf, ObjectType::byteValue),            Checksum.decoder(),            Decoder.ofLong(),            Decoder.ofLong()); diff --git a/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/DeltaMetaEntry.java b/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/DeltaMetaEntry.java index 8c6fd19..2be0426 100644 --- a/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/DeltaMetaEntry.java +++ b/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/DeltaMetaEntry.java @@ -5,6 +5,7 @@  package eu.mulk.jgvariant.ostree;  import eu.mulk.jgvariant.core.Decoder; +import java.io.ByteArrayOutputStream;  import java.nio.ByteBuffer;  import java.nio.ByteOrder;  import java.util.ArrayList; @@ -38,7 +39,9 @@ public record DeltaMetaEntry(      private static final Decoder<DeltaObject> DECODER =          Decoder.ofStructure( -            DeltaObject.class, Decoder.ofByte().map(ObjectType::valueOf), Checksum.decoder()); +            DeltaObject.class, +            Decoder.ofByte().map(ObjectType::valueOf, ObjectType::byteValue), +            Checksum.decoder());      /**       * Acquires a {@link Decoder} for the enclosing type. @@ -57,7 +60,19 @@ public record DeltaMetaEntry(            Checksum.decoder(),            Decoder.ofLong(),            Decoder.ofLong(), -          Decoder.ofByteArray().map(DeltaMetaEntry::parseObjectList)); +          Decoder.ofByteArray() +              .map(DeltaMetaEntry::parseObjectList, DeltaMetaEntry::serializeObjectList)); + +  private static byte[] serializeObjectList(List<DeltaObject> deltaObjects) { +    var output = new ByteArrayOutputStream(); + +    for (var deltaObject : deltaObjects) { +      output.write(deltaObject.objectType.byteValue()); +      output.writeBytes(deltaObject.checksum.byteString().bytes()); +    } + +    return output.toByteArray(); +  }    private static List<DeltaObject> parseObjectList(byte[] bytes) {      var byteBuffer = ByteBuffer.wrap(bytes); diff --git a/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/DeltaOperation.java b/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/DeltaOperation.java index 6edf217..d753a48 100644 --- a/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/DeltaOperation.java +++ b/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/DeltaOperation.java @@ -4,6 +4,7 @@  package eu.mulk.jgvariant.ostree; +import java.io.ByteArrayOutputStream;  import java.nio.ByteBuffer;  /** An operation in a static delta. */ @@ -67,6 +68,42 @@ public sealed interface DeltaOperation {      };    } +  default void writeTo(ByteArrayOutputStream output) { +    if (this instanceof OpenSpliceAndCloseMeta openSpliceAndCloseMeta) { +      output.write(DeltaOperationType.OPEN_SPLICE_AND_CLOSE.byteValue()); +      writeVarint64(output, openSpliceAndCloseMeta.offset); +      writeVarint64(output, openSpliceAndCloseMeta.size); +    } else if (this instanceof OpenSpliceAndCloseReal openSpliceAndCloseReal) { +      output.write(DeltaOperationType.OPEN_SPLICE_AND_CLOSE.byteValue()); +      writeVarint64(output, openSpliceAndCloseReal.modeOffset); +      writeVarint64(output, openSpliceAndCloseReal.xattrOffset); +      writeVarint64(output, openSpliceAndCloseReal.size); +      writeVarint64(output, openSpliceAndCloseReal.offset); +    } else if (this instanceof Open open) { +      output.write(DeltaOperationType.OPEN.byteValue()); +      writeVarint64(output, open.modeOffset); +      writeVarint64(output, open.xattrOffset); +      writeVarint64(output, open.size); +    } else if (this instanceof Write write) { +      output.write(DeltaOperationType.WRITE.byteValue()); +      writeVarint64(output, write.size); +      writeVarint64(output, write.offset); +    } else if (this instanceof SetReadSource setReadSource) { +      output.write(DeltaOperationType.SET_READ_SOURCE.byteValue()); +      writeVarint64(output, setReadSource.offset); +    } else if (this instanceof UnsetReadSource) { +      output.write(DeltaOperationType.UNSET_READ_SOURCE.byteValue()); +    } else if (this instanceof Close) { +      output.write(DeltaOperationType.CLOSE.byteValue()); +    } else if (this instanceof BsPatch bsPatch) { +      output.write(DeltaOperationType.BSPATCH.byteValue()); +      writeVarint64(output, bsPatch.offset); +      writeVarint64(output, bsPatch.size); +    } else { +      throw new IllegalStateException("unrecognized delta operation: %s".formatted(this)); +    } +  } +    /**     * Reads a Protobuf varint from a byte buffer.     * @@ -86,4 +123,23 @@ public sealed interface DeltaOperation {      return acc;    } + +  /** +   * Writes a Protobuf varint to an output stream. +   * +   * @see #readVarint64 +   */ +  private static void writeVarint64(ByteArrayOutputStream output, long value) { +    while (value != 0) { +      byte b = (byte) (value & 0x7F); +      value >>= 7; +      if (value != 0) { +        b |= (byte) 0x80; +      } +      output.write(b); +      if (value == 0) { +        break; +      } +    } +  }  } diff --git a/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/DeltaPartPayload.java b/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/DeltaPartPayload.java index f89d414..31c192d 100644 --- a/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/DeltaPartPayload.java +++ b/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/DeltaPartPayload.java @@ -8,12 +8,14 @@ import eu.mulk.jgvariant.core.Decoder;  import java.io.ByteArrayInputStream;  import java.io.ByteArrayOutputStream;  import java.io.IOException; -import java.io.UncheckedIOException;  import java.nio.ByteBuffer;  import java.nio.ByteOrder; +import java.nio.channels.Channels;  import java.util.ArrayList;  import java.util.List; +import org.tukaani.xz.LZMA2Options;  import org.tukaani.xz.XZInputStream; +import org.tukaani.xz.XZOutputStream;  /**   * A payload file from a static delta. @@ -36,7 +38,7 @@ public record DeltaPartPayload(      ByteString rawDataSource,      List<DeltaOperation> operations) { -  private static ByteBuffer preparse(ByteBuffer byteBuffer) { +  private static ByteBuffer decompress(ByteBuffer byteBuffer) {      byte compressionByte = byteBuffer.get(0);      var dataSlice = byteBuffer.slice(1, byteBuffer.limit() - 1);      return switch (compressionByte) { @@ -53,7 +55,7 @@ public record DeltaPartPayload(            yield ByteBuffer.wrap(decompressedOutputStream.toByteArray());          } catch (IOException e) {            // impossible -          throw new UncheckedIOException(e); +          throw new IllegalStateException(e);          }        }        default -> throw new IllegalArgumentException( @@ -61,10 +63,42 @@ public record DeltaPartPayload(      };    } +  private static ByteBuffer compress(ByteBuffer dataSlice) { +    var dataBytes = new byte[dataSlice.limit()]; +    dataSlice.get(dataBytes); +    var compressedOutputStream = new ByteArrayOutputStream(); + +    byte compressionByte = 'x'; +    compressedOutputStream.write(compressionByte); + +    try (var compressingOutputStream = +            new XZOutputStream(compressedOutputStream, new LZMA2Options()); +        var compressingChannel = Channels.newChannel(compressingOutputStream)) { +      compressingChannel.write(dataSlice); +      compressingOutputStream.write(dataBytes); +    } catch (IOException e) { +      // impossible +      throw new IllegalStateException(e); +    } + +    var compressedBytes = compressedOutputStream.toByteArray(); +    return ByteBuffer.wrap(compressedBytes); +  } + +  private static byte[] serializeDeltaOperationList(List<DeltaOperation> deltaOperations) { +    var output = new ByteArrayOutputStream(); + +    for (var currentOperation : deltaOperations) { +      currentOperation.writeTo(output); +    } + +    return output.toByteArray(); +  } +    private static List<DeltaOperation> parseDeltaOperationList( -      ByteString byteString, List<ObjectType> objectTypes) { +      byte[] bytes, List<ObjectType> objectTypes) {      List<DeltaOperation> deltaOperations = new ArrayList<>(); -    var byteBuffer = ByteBuffer.wrap(byteString.bytes()); +    var byteBuffer = ByteBuffer.wrap(bytes);      int objectIndex = 0;      while (byteBuffer.hasRemaining()) { @@ -119,8 +153,10 @@ public record DeltaPartPayload(              Decoder.ofArray(FileMode.decoder()),              Decoder.ofArray(Decoder.ofArray(Xattr.decoder())),              ByteString.decoder(), -            ByteString.decoder() -                .map(byteString -> parseDeltaOperationList(byteString, objectTypes))) -        .contramap(DeltaPartPayload::preparse); +            Decoder.ofByteArray() +                .map( +                    bytes -> parseDeltaOperationList(bytes, objectTypes), +                    deltaOperations -> serializeDeltaOperationList(deltaOperations))) +        .contramap(DeltaPartPayload::decompress, DeltaPartPayload::compress);    }  } diff --git a/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/DeltaSuperblock.java b/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/DeltaSuperblock.java index 50da203..9513fa0 100644 --- a/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/DeltaSuperblock.java +++ b/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/DeltaSuperblock.java @@ -5,10 +5,16 @@  package eu.mulk.jgvariant.ostree;  import eu.mulk.jgvariant.core.Decoder; +import eu.mulk.jgvariant.core.Signature; +import eu.mulk.jgvariant.core.Variant; +import java.io.ByteArrayOutputStream;  import java.nio.ByteBuffer;  import java.nio.ByteOrder; +import java.text.ParseException;  import java.util.ArrayList; +import java.util.HashMap;  import java.util.List; +import java.util.Map;  /**   * A static delta. @@ -67,10 +73,34 @@ public record DeltaSuperblock(                Checksum.decoder(),                Checksum.decoder(),                Commit.decoder(), -              Decoder.ofByteArray().map(DeltaSuperblock::parseDeltaNameList), +              Decoder.ofByteArray() +                  .map( +                      DeltaSuperblock::parseDeltaNameList, DeltaSuperblock::serializeDeltaNameList),                Decoder.ofArray(DeltaMetaEntry.decoder()).withByteOrder(ByteOrder.LITTLE_ENDIAN),                Decoder.ofArray(DeltaFallback.decoder()).withByteOrder(ByteOrder.LITTLE_ENDIAN)) -          .map(DeltaSuperblock::byteSwappedIfBigEndian); +          .map(DeltaSuperblock::byteSwappedIfBigEndian, DeltaSuperblock::withSpecifiedByteOrder); + +  private DeltaSuperblock withSpecifiedByteOrder() { +    Map<String, Variant> extendedMetadataMap = new HashMap<>(metadata().fields()); + +    try { +      extendedMetadataMap.putIfAbsent( +          "ostree.endianness", new Variant(Signature.parse("y"), (byte) 'l')); +    } catch (ParseException e) { +      // impossible +      throw new IllegalStateException(e); +    } + +    return new DeltaSuperblock( +        new Metadata(extendedMetadataMap), +        timestamp, +        fromChecksum, +        toChecksum, +        commit, +        dependencies, +        entries, +        fallbacks); +  }    private DeltaSuperblock byteSwappedIfBigEndian() {      // Fix up the endianness of the 'entries' and 'fallbacks' fields, which have @@ -97,6 +127,17 @@ public record DeltaSuperblock(          fallbacks.stream().map(DeltaFallback::byteSwapped).toList());    } +  private static byte[] serializeDeltaNameList(List<DeltaName> deltaNames) { +    var output = new ByteArrayOutputStream(); + +    for (var deltaName : deltaNames) { +      output.writeBytes(deltaName.fromChecksum().byteString().bytes()); +      output.writeBytes(deltaName.toChecksum().byteString().bytes()); +    } + +    return output.toByteArray(); +  } +    private static List<DeltaName> parseDeltaNameList(byte[] bytes) {      var byteBuffer = ByteBuffer.wrap(bytes);      List<DeltaName> deltaNames = new ArrayList<>(); diff --git a/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/Metadata.java b/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/Metadata.java index 62f0331..f485be1 100644 --- a/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/Metadata.java +++ b/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/Metadata.java @@ -20,7 +20,8 @@ import java.util.Map;  public record Metadata(Map<String, Variant> fields) {    private static final Decoder<Metadata> DECODER = -      Decoder.ofDictionary(Decoder.ofString(UTF_8), Decoder.ofVariant()).map(Metadata::new); +      Decoder.ofDictionary(Decoder.ofString(UTF_8), Decoder.ofVariant()) +          .map(Metadata::new, Metadata::fields);    /**     * Acquires a {@link Decoder} for the enclosing type. diff --git a/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/SignedDelta.java b/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/SignedDelta.java index 827d5e4..1e1e58e 100644 --- a/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/SignedDelta.java +++ b/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/SignedDelta.java @@ -31,11 +31,18 @@ public record SignedDelta(        Decoder.ofStructure(            SignedDelta.class,            Decoder.ofLong().withByteOrder(ByteOrder.BIG_ENDIAN), -          ByteString.decoder().map(SignedDelta::decodeSuperblock), +          Decoder.ofByteArray().map(SignedDelta::decodeSuperblock, SignedDelta::encodeSuperblock),            Decoder.ofDictionary(Decoder.ofString(US_ASCII), Decoder.ofVariant())); -  private static DeltaSuperblock decodeSuperblock(ByteString byteString) { -    return DeltaSuperblock.decoder().decode(ByteBuffer.wrap(byteString.bytes())); +  private static DeltaSuperblock decodeSuperblock(byte[] bytes) { +    return DeltaSuperblock.decoder().decode(ByteBuffer.wrap(bytes)); +  } + +  private static byte[] encodeSuperblock(DeltaSuperblock deltaSuperblock) { +    var byteBuffer = DeltaSuperblock.decoder().encode(deltaSuperblock); +    byte[] bytes = new byte[byteBuffer.remaining()]; +    byteBuffer.get(bytes); +    return bytes;    }    /** diff --git a/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/SummarySignature.java b/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/SummarySignature.java index 3c88759..5834c91 100644 --- a/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/SummarySignature.java +++ b/jgvariant-ostree/src/main/java/eu/mulk/jgvariant/ostree/SummarySignature.java @@ -22,7 +22,8 @@ import java.util.Map;  public record SummarySignature(Map<String, Variant> signatures) {    private static final Decoder<SummarySignature> DECODER = -      Decoder.ofDictionary(Decoder.ofString(UTF_8), Decoder.ofVariant()).map(SummarySignature::new); +      Decoder.ofDictionary(Decoder.ofString(UTF_8), Decoder.ofVariant()) +          .map(SummarySignature::new, SummarySignature::signatures);    /**     * Acquires a {@link Decoder} for the enclosing type. | 
