Merge pull request #423 from adriancole/optimize-bitset

Change header table to array and track references with long

okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/HpackDraft05.java

@@ -5,12 +5,19 @@ import java.io.DataInputStream;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.util.ArrayList;
-import java.util.BitSet;
+import java.util.Arrays;
 import java.util.List;
 
 /**
  * Read and write HPACK v05.
+ *
  * http://tools.ietf.org/html/draft-ietf-httpbis-header-compression-05
+ *
+ * This implementation uses an array for the header table with a bitset for
+ * references.  Dynamic entries are added to the array, starting in the last
+ * position moving forward.  When the array fills, it is doubled, up to the
+ * supported maximum of 64 headers.  HTTP requests or responses that require
+ * more than 64 headers are hence not currently supported.
  */
 final class HpackDraft05 {
 
@@ -19,35 +26,19 @@ final class HpackDraft05 {
     final ByteString name;
     final ByteString value;
     final int size;
-    // Static entries can be shared safely, as long as {@code referenced} is not mutated.
-    final boolean isStatic;
-    // Only read when in headerTable.
-    // Mutable to avoid needing another BitSet for referenced header indexes.  Using a BitSet for
-    // reference entries sounds good, except that entries are added at index zero.  This implies
-    // shifting the BitSet, which would be expensive to implement.
-    boolean referenced = true;
 
-    HeaderEntry(ByteString name, ByteString value, boolean isStatic) {
-      this(name, value, 32 + name.size() + value.size(), isStatic);
+    HeaderEntry(String name, String value) {
+      this(ByteString.encodeUtf8(name), ByteString.encodeUtf8(value));
     }
 
-    private HeaderEntry(ByteString name, ByteString value, int size, boolean isStatic) {
+    HeaderEntry(ByteString name, ByteString value) {
+      this(name, value, 32 + name.size() + value.size());
+    }
+
+    private HeaderEntry(ByteString name, ByteString value, int size) {
       this.name = name;
       this.value = value;
       this.size = size;
-      this.isStatic = isStatic;
-    }
-
-    /** Adds name and value, if this entry is referenced. */
-    void addTo(List<ByteString> out) {
-      if (!referenced) return;
-      out.add(name);
-      out.add(value);
-    }
-
-    /** Copies this header entry and designates it as not a static entry. */
-    @Override public HeaderEntry clone() {
-      return new HeaderEntry(name, value, size, false);
     }
   }
 
@@ -56,66 +47,66 @@ final class HpackDraft05 {
   private static final int PREFIX_8_BITS = 0xff;
 
   private static final HeaderEntry[] STATIC_HEADER_TABLE = new HeaderEntry[] {
-      staticEntry(":authority", ""),
-      staticEntry(":method", "GET"),
-      staticEntry(":method", "POST"),
-      staticEntry(":path", "/"),
-      staticEntry(":path", "/index.html"),
-      staticEntry(":scheme", "http"),
-      staticEntry(":scheme", "https"),
-      staticEntry(":status", "200"),
-      staticEntry(":status", "500"),
-      staticEntry(":status", "404"),
-      staticEntry(":status", "403"),
-      staticEntry(":status", "400"),
-      staticEntry(":status", "401"),
-      staticEntry("accept-charset", ""),
-      staticEntry("accept-encoding", ""),
-      staticEntry("accept-language", ""),
-      staticEntry("accept-ranges", ""),
-      staticEntry("accept", ""),
-      staticEntry("access-control-allow-origin", ""),
-      staticEntry("age", ""),
-      staticEntry("allow", ""),
-      staticEntry("authorization", ""),
-      staticEntry("cache-control", ""),
-      staticEntry("content-disposition", ""),
-      staticEntry("content-encoding", ""),
-      staticEntry("content-language", ""),
-      staticEntry("content-length", ""),
-      staticEntry("content-location", ""),
-      staticEntry("content-range", ""),
-      staticEntry("content-type", ""),
-      staticEntry("cookie", ""),
-      staticEntry("date", ""),
-      staticEntry("etag", ""),
-      staticEntry("expect", ""),
-      staticEntry("expires", ""),
-      staticEntry("from", ""),
-      staticEntry("host", ""),
-      staticEntry("if-match", ""),
-      staticEntry("if-modified-since", ""),
-      staticEntry("if-none-match", ""),
-      staticEntry("if-range", ""),
-      staticEntry("if-unmodified-since", ""),
-      staticEntry("last-modified", ""),
-      staticEntry("link", ""),
-      staticEntry("location", ""),
-      staticEntry("max-forwards", ""),
-      staticEntry("proxy-authenticate", ""),
-      staticEntry("proxy-authorization", ""),
-      staticEntry("range", ""),
-      staticEntry("referer", ""),
-      staticEntry("refresh", ""),
-      staticEntry("retry-after", ""),
-      staticEntry("server", ""),
-      staticEntry("set-cookie", ""),
-      staticEntry("strict-transport-security", ""),
-      staticEntry("transfer-encoding", ""),
-      staticEntry("user-agent", ""),
-      staticEntry("vary", ""),
-      staticEntry("via", ""),
-      staticEntry("www-authenticate", "")
+      new HeaderEntry(":authority", ""),
+      new HeaderEntry(":method", "GET"),
+      new HeaderEntry(":method", "POST"),
+      new HeaderEntry(":path", "/"),
+      new HeaderEntry(":path", "/index.html"),
+      new HeaderEntry(":scheme", "http"),
+      new HeaderEntry(":scheme", "https"),
+      new HeaderEntry(":status", "200"),
+      new HeaderEntry(":status", "500"),
+      new HeaderEntry(":status", "404"),
+      new HeaderEntry(":status", "403"),
+      new HeaderEntry(":status", "400"),
+      new HeaderEntry(":status", "401"),
+      new HeaderEntry("accept-charset", ""),
+      new HeaderEntry("accept-encoding", ""),
+      new HeaderEntry("accept-language", ""),
+      new HeaderEntry("accept-ranges", ""),
+      new HeaderEntry("accept", ""),
+      new HeaderEntry("access-control-allow-origin", ""),
+      new HeaderEntry("age", ""),
+      new HeaderEntry("allow", ""),
+      new HeaderEntry("authorization", ""),
+      new HeaderEntry("cache-control", ""),
+      new HeaderEntry("content-disposition", ""),
+      new HeaderEntry("content-encoding", ""),
+      new HeaderEntry("content-language", ""),
+      new HeaderEntry("content-length", ""),
+      new HeaderEntry("content-location", ""),
+      new HeaderEntry("content-range", ""),
+      new HeaderEntry("content-type", ""),
+      new HeaderEntry("cookie", ""),
+      new HeaderEntry("date", ""),
+      new HeaderEntry("etag", ""),
+      new HeaderEntry("expect", ""),
+      new HeaderEntry("expires", ""),
+      new HeaderEntry("from", ""),
+      new HeaderEntry("host", ""),
+      new HeaderEntry("if-match", ""),
+      new HeaderEntry("if-modified-since", ""),
+      new HeaderEntry("if-none-match", ""),
+      new HeaderEntry("if-range", ""),
+      new HeaderEntry("if-unmodified-since", ""),
+      new HeaderEntry("last-modified", ""),
+      new HeaderEntry("link", ""),
+      new HeaderEntry("location", ""),
+      new HeaderEntry("max-forwards", ""),
+      new HeaderEntry("proxy-authenticate", ""),
+      new HeaderEntry("proxy-authorization", ""),
+      new HeaderEntry("range", ""),
+      new HeaderEntry("referer", ""),
+      new HeaderEntry("refresh", ""),
+      new HeaderEntry("retry-after", ""),
+      new HeaderEntry("server", ""),
+      new HeaderEntry("set-cookie", ""),
+      new HeaderEntry("strict-transport-security", ""),
+      new HeaderEntry("transfer-encoding", ""),
+      new HeaderEntry("user-agent", ""),
+      new HeaderEntry("vary", ""),
+      new HeaderEntry("via", ""),
+      new HeaderEntry("www-authenticate", "")
   };
 
   private HpackDraft05() {
@@ -129,10 +120,25 @@ final class HpackDraft05 {
     private long bytesLeft = 0;
 
     // Visible for testing.
-    final List<HeaderEntry> headerTable = new ArrayList<HeaderEntry>(5); // average of 5 headers
-    final BitSet staticReferenceSet = new BitSet();
-    long headerTableSize = 0;
-    long maxHeaderTableSize = 4096; // TODO: needs to come from SETTINGS_HEADER_TABLE_SIZE.
+    HeaderEntry[] headerTable = new HeaderEntry[8]; // must be less than 64
+    // Array is populated back to front, so new entries always have lowest index.
+    int nextHeaderIndex = headerTable.length - 1;
+    int headerCount = 0;
+
+    /**
+     * Set bit positions indicate {@code headerTable[pos]} should be emitted.
+     */
+    // Using a long since the reference table < 64 entries.
+    long referencedHeaders = 0x0000000000000000L;
+
+    /**
+     * Set bit positions indicate {@code STATIC_HEADER_TABLE[pos]} should be
+     * emitted.
+     */
+    // Using a long since the static table < 64 entries.
+    long referencedStaticHeaders = 0x0000000000000000L;
+    int headerTableByteCount = 0;
+    int maxHeaderTableByteCount = 4096; // TODO: needs to come from SETTINGS_HEADER_TABLE_SIZE.
 
     Reader(DataInputStream in) {
       this.in = in;
@@ -149,51 +155,47 @@ final class HpackDraft05 {
       while (bytesLeft > 0) {
         int b = readByte();
 
-        if ((b & 0x80) != 0) {
+        if (b == 0x80) { // 10000000
+          clearReferenceSet();
+        } else if ((b & 0x80) == 0x80) { // 1NNNNNNN
           int index = readInt(b, PREFIX_7_BITS);
-          if (index == 0) {
-            clearReferenceSet();
+          readIndexedHeader(index - 1);
+        } else { // 0NNNNNNN
+          if (b == 0x40) { // 01000000
+            readLiteralHeaderWithoutIndexingNewName();
+          } else if ((b & 0xe0) == 0x40) {  // 01NNNNNN
+            int index = readInt(b, PREFIX_6_BITS);
+            readLiteralHeaderWithoutIndexingIndexedName(index - 1);
+          } else if (b == 0) { // 00000000
+            readLiteralHeaderWithIncrementalIndexingNewName();
+          } else if ((b & 0xc0) == 0) { // 00NNNNNN
+            int index = readInt(b, PREFIX_6_BITS);
+            readLiteralHeaderWithIncrementalIndexingIndexedName(index - 1);
           } else {
-            readIndexedHeader(index - 1);
+            // TODO: we should throw something that we can coerce to a PROTOCOL_ERROR
+            throw new AssertionError("unhandled byte: " + Integer.toBinaryString(b));
           }
-        } else if (b == 0x40) {
-          readLiteralHeaderWithoutIndexingNewName();
-        } else if ((b & 0xe0) == 0x40) {
-          int index = readInt(b, PREFIX_6_BITS);
-          readLiteralHeaderWithoutIndexingIndexedName(index - 1);
-        } else if (b == 0) {
-          readLiteralHeaderWithIncrementalIndexingNewName();
-        } else if ((b & 0xc0) == 0) {
-          int index = readInt(b, PREFIX_6_BITS);
-          readLiteralHeaderWithIncrementalIndexingIndexedName(index - 1);
-        } else {
-          // TODO: we should throw something that we can coerce to a PROTOCOL_ERROR
-          throw new AssertionError("unhandled byte: " + Integer.toBinaryString(b));
         }
       }
     }
 
     private void clearReferenceSet() {
-      staticReferenceSet.clear();
-      for (int i = 0, size = headerTable.size(); i < size; i++) {
-        HeaderEntry entry = headerTable.get(i);
-        if (entry.isStatic) { // lazy clone static entries on mutation.
-          entry = entry.clone();
-          entry.referenced = false;
-          headerTable.set(i, entry);
-        } else {
-          entry.referenced = false;
-        }
-      }
+      referencedStaticHeaders = 0x0000000000000000L;
+      referencedHeaders = 0x0000000000000000L;
     }
 
     public void emitReferenceSet() {
-      for (int i = staticReferenceSet.nextSetBit(0); i != -1;
-          i = staticReferenceSet.nextSetBit(i + 1)) {
-        STATIC_HEADER_TABLE[i].addTo(emittedHeaders);
+      for (int i = 0; i < STATIC_HEADER_TABLE.length; ++i) {
+        if (bitPositionSet(referencedStaticHeaders, i)) {
+          emittedHeaders.add(STATIC_HEADER_TABLE[i].name);
+          emittedHeaders.add(STATIC_HEADER_TABLE[i].value);
+        }
       }
-      for (int i = headerTable.size() - 1; i != -1; i--) {
-        headerTable.get(i).addTo(emittedHeaders);
+      for (int i = headerTable.length - 1; i != nextHeaderIndex; --i) {
+        if (bitPositionSet(referencedHeaders, i)) {
+          emittedHeaders.add(headerTable[i].name);
+          emittedHeaders.add(headerTable[i].value);
+        }
       }
     }
 
@@ -208,33 +210,36 @@ final class HpackDraft05 {
     }
 
     private void readIndexedHeader(int index) {
+
       if (isStaticHeader(index)) {
-        if (maxHeaderTableSize == 0) {
-          staticReferenceSet.set(index - headerTable.size());
+        if (maxHeaderTableByteCount == 0) {
+          // Set bit designating this static entry is referenced.
+          referencedStaticHeaders |= (1L << (index - headerCount));
         } else {
-          HeaderEntry staticEntry = STATIC_HEADER_TABLE[index - headerTable.size()];
+          HeaderEntry staticEntry = STATIC_HEADER_TABLE[index - headerCount];
           insertIntoHeaderTable(-1, staticEntry);
-       }
-      } else if (!headerTable.get(index).referenced) {
-        HeaderEntry existing = headerTable.get(index);
-        existing.referenced = true;
-        insertIntoHeaderTable(index, existing);
+        }
+      } else if (!bitPositionSet(referencedHeaders, headerTableIndex(index))) {
+        referencedHeaders |= (1L << headerTableIndex(index));
       } else {
         // TODO: we should throw something that we can coerce to a PROTOCOL_ERROR
         throw new AssertionError("invalid index " + index);
       }
     }
 
-    private void readLiteralHeaderWithoutIndexingIndexedName(int index)
-        throws IOException {
+    // referencedHeaders is relative to nextHeaderIndex + 1.
+    private int headerTableIndex(int index) {
+      return nextHeaderIndex + 1 + index;
+    }
+
+    private void readLiteralHeaderWithoutIndexingIndexedName(int index) throws IOException {
       ByteString name = getName(index);
       ByteString value = readString();
       emittedHeaders.add(name);
       emittedHeaders.add(value);
     }
 
-    private void readLiteralHeaderWithoutIndexingNewName()
-        throws IOException {
+    private void readLiteralHeaderWithoutIndexingNewName() throws IOException {
       ByteString name = readString();
       ByteString value = readString();
       emittedHeaders.add(name);
@@ -245,60 +250,89 @@ final class HpackDraft05 {
         throws IOException {
       ByteString name = getName(nameIndex);
       ByteString value = readString();
-      insertIntoHeaderTable(-1, new HeaderEntry(name, value, false));
+      insertIntoHeaderTable(-1, new HeaderEntry(name, value));
     }
 
     private void readLiteralHeaderWithIncrementalIndexingNewName() throws IOException {
       ByteString name = readString();
       ByteString value = readString();
-      insertIntoHeaderTable(-1, new HeaderEntry(name, value, false));
+      insertIntoHeaderTable(-1, new HeaderEntry(name, value));
     }
 
     private ByteString getName(int index) {
       if (isStaticHeader(index)) {
-        return STATIC_HEADER_TABLE[index - headerTable.size()].name;
+        return STATIC_HEADER_TABLE[index - headerCount].name;
       } else {
-        return headerTable.get(index).name;
+        return headerTable[headerTableIndex(index)].name;
       }
     }
 
     private boolean isStaticHeader(int index) {
-      return index >= headerTable.size();
+      return index >= headerCount;
     }
 
     /** index == -1 when new. */
     private void insertIntoHeaderTable(int index, HeaderEntry entry) {
       int delta = entry.size;
       if (index != -1) { // Index -1 == new header.
-        delta -= headerTable.get(index).size;
+        delta -= headerTable[headerTableIndex(index)].size;
       }
 
       // if the new or replacement header is too big, drop all entries.
-      if (delta > maxHeaderTableSize) {
-        staticReferenceSet.clear();
-        headerTable.clear();
-        headerTableSize = 0;
+      if (delta > maxHeaderTableByteCount) {
+        referencedStaticHeaders = 0x0000000000000000L;
+        referencedHeaders = 0x0000000000000000L;
+        Arrays.fill(headerTable, null);
+        nextHeaderIndex = headerTable.length - 1;
+        headerCount = 0;
+        headerTableByteCount = 0;
         // emit the large header to the callback.
-        entry.addTo(emittedHeaders);
+        emittedHeaders.add(entry.name);
+        emittedHeaders.add(entry.value);
         return;
       }
 
       // Evict headers to the required length.
-      while (headerTableSize + delta > maxHeaderTableSize) {
-        remove(headerTable.size() - 1);
+      int bytesToRecover = (headerTableByteCount + delta) - maxHeaderTableByteCount;
+      int entriesToEvict = 0;
+      if (bytesToRecover > 0) {
+        // determine how many headers need to be evicted.
+        for (int j = headerTable.length - 1; j >= nextHeaderIndex && bytesToRecover > 0; j--) {
+          bytesToRecover -= headerTable[j].size;
+          headerTableByteCount -= headerTable[j].size;
+          headerCount--;
+          entriesToEvict++;
+        }
+        // shift elements over
+        referencedHeaders = referencedHeaders << entriesToEvict;
+        System.arraycopy(headerTable, nextHeaderIndex + 1, headerTable,
+            nextHeaderIndex + 1 + entriesToEvict, headerCount);
+        nextHeaderIndex += entriesToEvict;
       }
 
       if (index == -1) {
-        headerTable.add(0, entry);
+        if (headerCount + 1 > headerTable.length) {
+          if (headerTable.length == 64) {
+            // We would need to switch off long to bitset to support > 64 headers.
+            throw new UnsupportedOperationException(
+                "Header tables with count > 64 not yet supported!");
+          }
+          HeaderEntry[] doubled = new HeaderEntry[headerTable.length * 2];
+          System.arraycopy(headerTable, 0, doubled, headerTable.length, headerTable.length);
+          referencedHeaders = referencedHeaders << headerTable.length;
+          nextHeaderIndex = headerTable.length - 1;
+          headerTable = doubled;
+        }
+        index = nextHeaderIndex--;
+        referencedHeaders |= (1L << index);
+        headerTable[index] = entry;
+        headerCount++;
       } else { // Replace value at same position.
-        headerTable.set(index, entry);
+        index += headerTableIndex(index) + entriesToEvict;
+        referencedHeaders |= (1L << index);
+        headerTable[index] = entry;
       }
-
-      headerTableSize += delta;
-    }
-
-    private void remove(int index) {
-      headerTableSize -= headerTable.remove(index).size;
+      headerTableByteCount += delta;
     }
 
     private int readByte() throws IOException {
@@ -335,13 +369,19 @@ final class HpackDraft05 {
     public ByteString readString() throws IOException {
       int firstByte = readByte();
       int length = readInt(firstByte, PREFIX_8_BITS);
-      byte[] encoded = new byte[length];
+      if ((length & 0x80) == 0x80) { // 1NNNNNNN
+        length &= ~0x80;
+        // TODO: actually decode huffman!
+      }
       bytesLeft -= length;
-      in.readFully(encoded);
-      return ByteString.of(encoded);
+      return ByteString.read(in, length);
     }
   }
 
+  static boolean bitPositionSet(long referenceBitSet, int i) {
+    return ((referenceBitSet >> i) & 1L) == 1;
+  }
+
   static class Writer {
     private final OutputStream out;
 
@@ -383,8 +423,4 @@ final class HpackDraft05 {
       data.write(out);
     }
   }
-
-  private static HeaderEntry staticEntry(String name, String value) {
-    return new HeaderEntry(ByteString.encodeUtf8(name), ByteString.encodeUtf8(value), true);
-  }
 }

okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/Http20Draft09.java

@@ -150,7 +150,7 @@ public final class Http20Draft09 implements Variant {
           return true;
       }
 
-      throw new UnsupportedOperationException("TODO");
+      throw new UnsupportedOperationException(Integer.toBinaryString(type));
     }
 
     private void readHeaders(Handler handler, int flags, int length, int streamId)

okhttp-protocols/src/test/java/com/squareup/okhttp/internal/spdy/HpackDraft05Test.java

@@ -27,7 +27,11 @@ import org.junit.Before;
 import org.junit.Test;
 
 import static com.squareup.okhttp.internal.Util.byteStringList;
+import static com.squareup.okhttp.internal.spdy.HpackDraft05.bitPositionSet;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 public class HpackDraft05Test {
 
@@ -43,18 +47,137 @@ public class HpackDraft05Test {
    * Ensure the larger header content is not lost.
    */
   @Test public void tooLargeToHPackIsStillEmitted() throws IOException {
-    char[] tooLarge = new char[4096];
-    Arrays.fill(tooLarge, 'a');
-    final List<ByteString> sentHeaders = byteStringList("foo", new String(tooLarge));
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+    out.write(0x00); // Literal indexed
+    out.write(0x0a); // Literal name (len = 10)
+    out.write("custom-key".getBytes(), 0, 10);
+
+    out.write(0x0d); // Literal value (len = 13)
+    out.write("custom-header".getBytes(), 0, 13);
+
+    bytesIn.set(out.toByteArray());
+    hpackReader.maxHeaderTableByteCount = 1;
+    hpackReader.readHeaders(out.size());
+    hpackReader.emitReferenceSet();
+
+    assertEquals(0, hpackReader.headerCount);
+
+    assertEquals(byteStringList("custom-key", "custom-header"), hpackReader.getAndReset());
+  }
+
+  /** Oldest entries are evicted to support newer ones. */
+  @Test public void testEviction() throws IOException {
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+    out.write(0x00); // Literal indexed
+    out.write(0x0a); // Literal name (len = 10)
+    out.write("custom-foo".getBytes(), 0, 10);
+
+    out.write(0x0d); // Literal value (len = 13)
+    out.write("custom-header".getBytes(), 0, 13);
+
+    out.write(0x00); // Literal indexed
+    out.write(0x0a); // Literal name (len = 10)
+    out.write("custom-bar".getBytes(), 0, 10);
+
+    out.write(0x0d); // Literal value (len = 13)
+    out.write("custom-header".getBytes(), 0, 13);
+
+    out.write(0x00); // Literal indexed
+    out.write(0x0a); // Literal name (len = 10)
+    out.write("custom-baz".getBytes(), 0, 10);
+
+    out.write(0x0d); // Literal value (len = 13)
+    out.write("custom-header".getBytes(), 0, 13);
+
+    bytesIn.set(out.toByteArray());
+    hpackReader.maxHeaderTableByteCount = 110;
+    hpackReader.readHeaders(out.size());
+    hpackReader.emitReferenceSet();
+
+    assertEquals(2, hpackReader.headerCount);
+
+    HpackDraft05.HeaderEntry entry = hpackReader.headerTable[headerTableLength() - 1];
+    checkEntry(entry, "custom-bar", "custom-header", 55);
+    assertHeaderReferenced(headerTableLength() - 1);
+
+    entry = hpackReader.headerTable[headerTableLength() - 2];
+    checkEntry(entry, "custom-baz", "custom-header", 55);
+    assertHeaderReferenced(headerTableLength() - 2);
+
+    // foo isn't here as it is no longer in the table.
+    // TODO: emit before eviction?
+    assertEquals(byteStringList("custom-bar", "custom-header", "custom-baz", "custom-header"),
+        hpackReader.getAndReset());
+  }
+
+  /** Header table backing array is initially 8 long, let's ensure it grows. */
+  @Test public void dynamicallyGrowsUpTo64Entries() throws IOException {
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+    for (int i = 0; i < 64; i++) {
+      out.write(0x00); // Literal indexed
+      out.write(0x0a); // Literal name (len = 10)
+      out.write("custom-foo".getBytes(), 0, 10);
+
+      out.write(0x0d); // Literal value (len = 13)
+      out.write("custom-header".getBytes(), 0, 13);
+    }
 
-    ByteArrayOutputStream out = literalHeaders(sentHeaders);
     bytesIn.set(out.toByteArray());
     hpackReader.readHeaders(out.size());
     hpackReader.emitReferenceSet();
 
-    assertEquals(0, hpackReader.headerTable.size());
+    assertEquals(64, hpackReader.headerCount);
+  }
 
-    assertEquals(sentHeaders, hpackReader.getAndReset());
+  @Test public void greaterThan64HeadersNotYetSupported() throws IOException {
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+    for (int i = 0; i < 65; i++) {
+      out.write(0x00); // Literal indexed
+      out.write(0x0a); // Literal name (len = 10)
+      out.write("custom-foo".getBytes(), 0, 10);
+
+      out.write(0x0d); // Literal value (len = 13)
+      out.write("custom-header".getBytes(), 0, 13);
+    }
+
+    bytesIn.set(out.toByteArray());
+    try {
+      hpackReader.readHeaders(out.size());
+      fail();
+    } catch (UnsupportedOperationException expected) {
+    }
+  }
+
+  /** Huffman headers are accepted, but come out as garbage for now. */
+  @Test public void huffmanDecodingNotYetSupported() throws IOException {
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+    out.write(0x04); // == Literal indexed ==
+                     // Indexed name (idx = 4) -> :path
+    out.write(0x8b); // Literal value Huffman encoded 11 bytes
+                     // decodes to www.example.com which is length 15
+    byte[] huffmanBytes = new byte[] {
+        (byte) 0xdb, (byte) 0x6d, (byte) 0x88, (byte) 0x3e,
+        (byte) 0x68, (byte) 0xd1, (byte) 0xcb, (byte) 0x12,
+        (byte) 0x25, (byte) 0xba, (byte) 0x7f};
+    out.write(huffmanBytes, 0, huffmanBytes.length);
+
+    bytesIn.set(out.toByteArray());
+    hpackReader.readHeaders(out.size());
+    hpackReader.emitReferenceSet();
+
+    assertEquals(1, hpackReader.headerCount);
+    // this will change when we decode huffman
+    assertEquals(48, hpackReader.headerTableByteCount);
+
+    HpackDraft05.HeaderEntry entry = hpackReader.headerTable[headerTableLength() - 1];
+    // TODO: huffman bytes are not what we want!
+    checkEntry(entry, ":path", new String(huffmanBytes, "UTF-8"), 48);
+    assertHeaderReferenced(headerTableLength() - 1);
   }
 
   /**
@@ -74,11 +197,12 @@ public class HpackDraft05Test {
     hpackReader.readHeaders(out.size());
     hpackReader.emitReferenceSet();
 
-    assertEquals(1, hpackReader.headerTable.size());
-    assertEquals(55, hpackReader.headerTableSize);
+    assertEquals(1, hpackReader.headerCount);
+    assertEquals(55, hpackReader.headerTableByteCount);
 
-    HpackDraft05.HeaderEntry entry = hpackReader.headerTable.get(0);
-    checkEntry(entry, "custom-key", "custom-header", 55, true);
+    HpackDraft05.HeaderEntry entry = hpackReader.headerTable[headerTableLength() - 1];
+    checkEntry(entry, "custom-key", "custom-header", 55);
+    assertHeaderReferenced(headerTableLength() - 1);
 
     assertEquals(byteStringList("custom-key", "custom-header"), hpackReader.getAndReset());
   }
@@ -98,7 +222,7 @@ public class HpackDraft05Test {
     hpackReader.readHeaders(out.size());
     hpackReader.emitReferenceSet();
 
-    assertEquals(0, hpackReader.headerTable.size());
+    assertEquals(0, hpackReader.headerCount);
 
     assertEquals(byteStringList(":path", "/sample/path"), hpackReader.getAndReset());
   }
@@ -116,11 +240,12 @@ public class HpackDraft05Test {
     hpackReader.readHeaders(out.size());
     hpackReader.emitReferenceSet();
 
-    assertEquals(1, hpackReader.headerTable.size());
-    assertEquals(42, hpackReader.headerTableSize);
+    assertEquals(1, hpackReader.headerCount);
+    assertEquals(42, hpackReader.headerTableByteCount);
 
-    HpackDraft05.HeaderEntry entry = hpackReader.headerTable.get(0);
-    checkEntry(entry, ":method", "GET", 42, true);
+    HpackDraft05.HeaderEntry entry = hpackReader.headerTable[headerTableLength() - 1];
+    checkEntry(entry, ":method", "GET", 42);
+    assertHeaderReferenced(headerTableLength() - 1);
 
     assertEquals(byteStringList(":method", "GET"), hpackReader.getAndReset());
   }
@@ -135,12 +260,12 @@ public class HpackDraft05Test {
                      // idx = 2 -> :method: GET
 
     bytesIn.set(out.toByteArray());
-    hpackReader.maxHeaderTableSize = 0; // SETTINGS_HEADER_TABLE_SIZE == 0
+    hpackReader.maxHeaderTableByteCount = 0; // SETTINGS_HEADER_TABLE_SIZE == 0
     hpackReader.readHeaders(out.size());
     hpackReader.emitReferenceSet();
 
     // Not buffered in header table.
-    assertEquals(0, hpackReader.headerTable.size());
+    assertEquals(0, hpackReader.headerCount);
 
     assertEquals(byteStringList(":method", "GET"), hpackReader.getAndReset());
   }
@@ -186,26 +311,30 @@ public class HpackDraft05Test {
   }
 
   private void checkFirstRequestWithoutHuffman() {
-    assertEquals(4, hpackReader.headerTable.size());
+    assertEquals(4, hpackReader.headerCount);
 
     // [  1] (s =  57) :authority: www.example.com
-    HpackDraft05.HeaderEntry entry = hpackReader.headerTable.get(0);
-    checkEntry(entry, ":authority", "www.example.com", 57, true);
+    HpackDraft05.HeaderEntry entry = hpackReader.headerTable[headerTableLength() - 4];
+    checkEntry(entry, ":authority", "www.example.com", 57);
+    assertHeaderReferenced(headerTableLength() - 4);
 
     // [  2] (s =  38) :path: /
-    entry = hpackReader.headerTable.get(1);
-    checkEntry(entry, ":path", "/", 38, true);
+    entry = hpackReader.headerTable[headerTableLength() - 3];
+    checkEntry(entry, ":path", "/", 38);
+    assertHeaderReferenced(headerTableLength() - 3);
 
     // [  3] (s =  43) :scheme: http
-    entry = hpackReader.headerTable.get(2);
-    checkEntry(entry, ":scheme", "http", 43, true);
+    entry = hpackReader.headerTable[headerTableLength() - 2];
+    checkEntry(entry, ":scheme", "http", 43);
+    assertHeaderReferenced(headerTableLength() - 2);
 
     // [  4] (s =  42) :method: GET
-    entry = hpackReader.headerTable.get(3);
-    checkEntry(entry, ":method", "GET", 42, true);
+    entry = hpackReader.headerTable[headerTableLength() - 1];
+    checkEntry(entry, ":method", "GET", 42);
+    assertHeaderReferenced(headerTableLength() - 1);
 
     // Table size: 180
-    assertEquals(180, hpackReader.headerTableSize);
+    assertEquals(180, hpackReader.headerTableByteCount);
 
     // Decoded header set:
     assertEquals(byteStringList(
@@ -227,30 +356,35 @@ public class HpackDraft05Test {
   }
 
   private void checkSecondRequestWithoutHuffman() {
-    assertEquals(5, hpackReader.headerTable.size());
+    assertEquals(5, hpackReader.headerCount);
 
     // [  1] (s =  53) cache-control: no-cache
-    HpackDraft05.HeaderEntry entry = hpackReader.headerTable.get(0);
-    checkEntry(entry, "cache-control", "no-cache", 53, true);
+    HpackDraft05.HeaderEntry entry = hpackReader.headerTable[headerTableLength() - 5];
+    checkEntry(entry, "cache-control", "no-cache", 53);
+    assertHeaderReferenced(headerTableLength() - 5);
 
     // [  2] (s =  57) :authority: www.example.com
-    entry = hpackReader.headerTable.get(1);
-    checkEntry(entry, ":authority", "www.example.com", 57, true);
+    entry = hpackReader.headerTable[headerTableLength() - 4];
+    checkEntry(entry, ":authority", "www.example.com", 57);
+    assertHeaderReferenced(headerTableLength() - 4);
 
     // [  3] (s =  38) :path: /
-    entry = hpackReader.headerTable.get(2);
-    checkEntry(entry, ":path", "/", 38, true);
+    entry = hpackReader.headerTable[headerTableLength() - 3];
+    checkEntry(entry, ":path", "/", 38);
+    assertHeaderReferenced(headerTableLength() - 3);
 
     // [  4] (s =  43) :scheme: http
-    entry = hpackReader.headerTable.get(3);
-    checkEntry(entry, ":scheme", "http", 43, true);
+    entry = hpackReader.headerTable[headerTableLength() - 2];
+    checkEntry(entry, ":scheme", "http", 43);
+    assertHeaderReferenced(headerTableLength() - 2);
 
     // [  5] (s =  42) :method: GET
-    entry = hpackReader.headerTable.get(4);
-    checkEntry(entry, ":method", "GET", 42, true);
+    entry = hpackReader.headerTable[headerTableLength() - 1];
+    checkEntry(entry, ":method", "GET", 42);
+    assertHeaderReferenced(headerTableLength() - 1);
 
     // Table size: 233
-    assertEquals(233, hpackReader.headerTableSize);
+    assertEquals(233, hpackReader.headerTableByteCount);
 
     // Decoded header set:
     assertEquals(byteStringList(
@@ -283,42 +417,50 @@ public class HpackDraft05Test {
   }
 
   private void checkThirdRequestWithoutHuffman() {
-    assertEquals(8, hpackReader.headerTable.size());
+    assertEquals(8, hpackReader.headerCount);
 
     // [  1] (s =  54) custom-key: custom-value
-    HpackDraft05.HeaderEntry entry = hpackReader.headerTable.get(0);
-    checkEntry(entry, "custom-key", "custom-value", 54, true);
+    HpackDraft05.HeaderEntry entry = hpackReader.headerTable[headerTableLength() - 8];
+    checkEntry(entry, "custom-key", "custom-value", 54);
+    assertHeaderReferenced(headerTableLength() - 8);
 
     // [  2] (s =  48) :path: /index.html
-    entry = hpackReader.headerTable.get(1);
-    checkEntry(entry, ":path", "/index.html", 48, true);
+    entry = hpackReader.headerTable[headerTableLength() - 7];
+    checkEntry(entry, ":path", "/index.html", 48);
+    assertHeaderReferenced(headerTableLength() - 7);
 
     // [  3] (s =  44) :scheme: https
-    entry = hpackReader.headerTable.get(2);
-    checkEntry(entry, ":scheme", "https", 44, true);
+    entry = hpackReader.headerTable[headerTableLength() - 6];
+    checkEntry(entry, ":scheme", "https", 44);
+    assertHeaderReferenced(headerTableLength() - 6);
 
     // [  4] (s =  53) cache-control: no-cache
-    entry = hpackReader.headerTable.get(3);
-    checkEntry(entry, "cache-control", "no-cache", 53, false);
+    entry = hpackReader.headerTable[headerTableLength() - 5];
+    checkEntry(entry, "cache-control", "no-cache", 53);
+    assertHeaderNotReferenced(headerTableLength() - 5);
 
     // [  5] (s =  57) :authority: www.example.com
-    entry = hpackReader.headerTable.get(4);
-    checkEntry(entry, ":authority", "www.example.com", 57, true);
+    entry = hpackReader.headerTable[headerTableLength() - 4];
+    checkEntry(entry, ":authority", "www.example.com", 57);
+    assertHeaderReferenced(headerTableLength() - 4);
 
     // [  6] (s =  38) :path: /
-    entry = hpackReader.headerTable.get(5);
-    checkEntry(entry, ":path", "/", 38, false);
+    entry = hpackReader.headerTable[headerTableLength() - 3];
+    checkEntry(entry, ":path", "/", 38);
+    assertHeaderNotReferenced(headerTableLength() - 3);
 
     // [  7] (s =  43) :scheme: http
-    entry = hpackReader.headerTable.get(6);
-    checkEntry(entry, ":scheme", "http", 43, false);
+    entry = hpackReader.headerTable[headerTableLength() - 2];
+    checkEntry(entry, ":scheme", "http", 43);
+    assertHeaderNotReferenced(headerTableLength() - 2);
 
     // [  8] (s =  42) :method: GET
-    entry = hpackReader.headerTable.get(7);
-    checkEntry(entry, ":method", "GET", 42, true);
+    entry = hpackReader.headerTable[headerTableLength() - 1];
+    checkEntry(entry, ":method", "GET", 42);
+    assertHeaderReferenced(headerTableLength() - 1);
 
     // Table size: 379
-    assertEquals(379, hpackReader.headerTableSize);
+    assertEquals(379, hpackReader.headerTableByteCount);
 
     // Decoded header set:
     // TODO: order is not correct per docs, but then again, the spec doesn't require ordering.
@@ -331,7 +473,8 @@ public class HpackDraft05Test {
   }
 
   private ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
-  private final HpackDraft05.Writer hpackWriter = new HpackDraft05.Writer(new DataOutputStream(bytesOut));
+  private final HpackDraft05.Writer hpackWriter =
+      new HpackDraft05.Writer(new DataOutputStream(bytesOut));
 
   @Test public void readSingleByteInt() throws IOException {
     assertEquals(10, new HpackDraft05.Reader(byteStream()).readInt(10, 31));
@@ -415,12 +558,10 @@ public class HpackDraft05Test {
     return headerBytes;
   }
 
-  private void checkEntry(HpackDraft05.HeaderEntry entry, String name, String value, int size,
-      boolean referenced) {
+  private void checkEntry(HpackDraft05.HeaderEntry entry, String name, String value, int size) {
     assertEquals(name, entry.name.utf8());
     assertEquals(value, entry.value.utf8());
     assertEquals(size, entry.size);
-    assertEquals(referenced, entry.referenced);
   }
 
   private void assertBytes(int... bytes) {
@@ -438,6 +579,18 @@ public class HpackDraft05Test {
     return data;
   }
 
+  private void assertHeaderReferenced(int index) {
+    assertTrue(bitPositionSet(hpackReader.referencedHeaders, index));
+  }
+
+  private void assertHeaderNotReferenced(int index) {
+    assertFalse(bitPositionSet(hpackReader.referencedHeaders, index));
+  }
+
+  private int headerTableLength() {
+    return hpackReader.headerTable.length;
+  }
+
   private static class MutableByteArrayInputStream extends ByteArrayInputStream {
 
     private MutableByteArrayInputStream() {

okhttp/src/main/java/com/squareup/okhttp/internal/http/SpdyTransport.java

@@ -154,7 +154,7 @@ public final class SpdyTransport implements Transport {
       throw new IllegalArgumentException("Unexpected name value block: " + nameValueBlock);
     }
     String status = null;
-    String version = null;
+    String version = "HTTP/1.1"; // TODO: why are we expecting :version?
 
     Headers.Builder headersBuilder = new Headers.Builder();
     headersBuilder.set(OkHeaders.SELECTED_TRANSPORT, protocol);

okhttp/src/test/java/com/squareup/okhttp/internal/http/ExternalHttp2Example.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.squareup.okhttp.internal.http;
+
+import com.squareup.okhttp.OkHttpClient;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.URL;
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSession;
+
+public final class ExternalHttp2Example {
+  public static void main(String[] args) throws Exception {
+    URL url = new URL("https://twitter.com/");
+    HttpsURLConnection connection = (HttpsURLConnection) new OkHttpClient().open(url);
+
+    connection.setHostnameVerifier(new HostnameVerifier() {
+      @Override public boolean verify(String s, SSLSession sslSession) {
+        System.out.println("VERIFYING " + s);
+        return true;
+      }
+    });
+
+    int responseCode = connection.getResponseCode();
+    System.out.println(responseCode);
+
+    BufferedReader reader =
+        new BufferedReader(new InputStreamReader(connection.getInputStream(), "UTF-8"));
+    String line;
+    while ((line = reader.readLine()) != null) {
+      System.out.println(line);
+    }
+  }
+}