HttpClient 4 is not reading gzip content properly

Question

Hi,
I'm struggling with this issue when trying to handle some data from an Url. The returned content is ok (checked with wireshark), Content-Length header is set correctly too.
I've tried to debug and find the source of this problem, but it's somewhere deep inside the core classes. What may be worth mentioning is that the content length is > 1460 bytes (a single tcp frame) - actually, it's around 6-10 kbytes, so multiple frames will be read before reassembling the http response. Actually, the first frame looks ok everytime (I've inspected the hex dump). The problem comes, like I said before, when multiple tcp frames are required.
Has anyone encountered this problem? It would help a lot, thanks.
ps: no "deflate" encoding is available :(

Answer 1

I did a bit of debugging, I can reproduce the problem using the code below. You can copy-paste it and try for yourself.
My host:
OS :
Linux ... 2.6.31-14-generic #48-Ubuntu SMP Fri Oct 16 14:04:26 UTC 2009 i686 GNU/Linux
Java :
java version "1.6.0_21"
Java(TM) SE Runtime Environment (build 1.6.0_21-b06)
Java HotSpot(TM) Server VM (build 17.0-b16, mixed mode)
Output of the code below:
Entity present, length: 319793
0-bytes : 317097
Non 0-bytes : 2696
(also the byte-to-number mapping is displayed, but hidden in this post)
Hope this makes everything clear.
     public static void main (String[] args) throws IOException {
          HttpClient client = new DefaultHttpClient();
          HttpResponse resp     public static void main (String[] args) throws IOException {
          HttpClient client = new DefaultHttpClient();
          HttpResponse response = client.execute(new HttpGet("http://en.wikipedia.org/wiki/New_york"));
          HttpEntity entity = response.getEntity();
          printHttpEntity(entity);
     private static void printHttpEntity(HttpEntity entity) throws IOException {
          if (entity != null) {
               System.out.println ("Entity present, length: " + entity.getContentLength());
               // Read the whole contents at once
               byte[] contents = new byte [(int) entity.getContentLength()];
               entity.getContent().read(contents);
               int zeroByte = countZeroBytes(contents);
               System.out.println ("0-bytes : " + zeroByte);
               System.out.println ("Non 0-bytes : " + (contents.length - zeroByte));
               SortedMap<Byte, Integer> map = mapBytesToDistro(contents);
               for (Entry<Byte, Integer> e : map.entrySet()) {
                    System.out.println (e.getKey() + " => " + e.getValue());
     private static int countZeroBytes (byte[] contents) {
          int count = 0;
          for (int i = 0; i < contents.length; i++) {
               if (contents == 0)
                    count ++;
          return count;
     private static SortedMap<Byte, Integer> mapBytesToDistro (byte[] contents) {
          SortedMap<Byte, Integer> map = new TreeMap<Byte, Integer>();
          for (int i = 0; i < contents.length; i++) {
               if (!map.containsKey(contents [i]))
                    map.put(contents [i], new Integer(0));
               map.put(contents [i], map.get(contents [i]) + 1);
          return map;
     }onse = client.execute(new HttpGet("http://en.wikipedia.org/wiki/New_york"));
          HttpEntity entity = response.getEntity();
          printHttpEntity(entity);
     private static void printHttpEntity(HttpEntity entity) throws IOException {
          if (entity != null) {
               System.out.println ("Entity present, length: " + entity.getContentLength());
               // Read the whole contents at once
               byte[] contents = new byte [(int) entity.getContentLength()];
               entity.getContent().read(contents);
               int zeroByte = countZeroBytes(contents);
               System.out.println ("0-bytes : " + zeroByte);
               System.out.println ("Non 0-bytes : " + (contents.length - zeroByte));
               SortedMap<Byte, Integer> map = mapBytesToDistro(contents);
               for (Entry<Byte, Integer> e : map.entrySet()) {
                    System.out.println (e.getKey() + " => " + e.getValue());
     private static int countZeroBytes (byte[] contents) {
          int count = 0;
          for (int i = 0; i < contents.length; i++) {
               if (contents [i] == 0)
                    count ++;
          return count;
     private static SortedMap<Byte, Integer> mapBytesToDistro (byte[] contents) {
          SortedMap<Byte, Integer> map = new TreeMap<Byte, Integer>();
          for (int i = 0; i < contents.length; i++) {
               if (!map.containsKey(contents [i]))
                    map.put(contents [i], new Integer(0));
               map.put(contents [i], map.get(contents [i]) + 1);
          return map;