Faster library list downloading by downloading gzipped version

app/test/cc/arduino/contributions/GzippedJsonDownloaderTest.java

@@ -0,0 +1,53 @@
+package cc.arduino.contributions;
+
+import cc.arduino.contributions.libraries.LibrariesIndex;
+import cc.arduino.utils.MultiStepProgress;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.module.mrbean.MrBeanModule;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import processing.app.helpers.FileUtils;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.net.URL;
+
+import static org.junit.Assert.assertTrue;
+
+public class GzippedJsonDownloaderTest {
+
+  private File tempFolder;
+  private File tempFile;
+  private DownloadableContributionsDownloader downloader;
+
+  @Before
+  public void setUp() throws Exception {
+    tempFolder = FileUtils.createTempFolder();
+    tempFile = File.createTempFile("test", ".json");
+    downloader = new DownloadableContributionsDownloader(tempFolder);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    FileUtils.recursiveDelete(tempFolder);
+    FileUtils.recursiveDelete(tempFile);
+  }
+
+  @Test
+  public void testJsonDownload() throws Exception {
+    new GZippedJsonDownloader(downloader, new URL("http://downloads.arduino.cc/libraries/library_index.json"), new URL("http://downloads.arduino.cc/libraries/library_index.json.gz")).download(tempFile, new MultiStepProgress(1), "");
+
+    InputStream indexIn = new FileInputStream(tempFile);
+    ObjectMapper mapper = new ObjectMapper();
+    mapper.registerModule(new MrBeanModule());
+    mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true);
+    mapper.configure(DeserializationFeature.EAGER_DESERIALIZER_FETCH, true);
+    mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+    LibrariesIndex librariesIndex = mapper.readValue(indexIn, LibrariesIndex.class);
+
+    assertTrue(librariesIndex != null);
+  }
+}

app/test/cc/arduino/contributions/JsonDownloaderTest.java

@@ -0,0 +1,53 @@
+package cc.arduino.contributions;
+
+import cc.arduino.contributions.libraries.LibrariesIndex;
+import cc.arduino.utils.MultiStepProgress;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.module.mrbean.MrBeanModule;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import processing.app.helpers.FileUtils;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.net.URL;
+
+import static org.junit.Assert.assertTrue;
+
+public class JsonDownloaderTest {
+
+  private File tempFolder;
+  private File tempFile;
+  private DownloadableContributionsDownloader downloader;
+
+  @Before
+  public void setUp() throws Exception {
+    tempFolder = FileUtils.createTempFolder();
+    tempFile = File.createTempFile("test", ".json");
+    downloader = new DownloadableContributionsDownloader(tempFolder);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    FileUtils.recursiveDelete(tempFolder);
+    FileUtils.recursiveDelete(tempFile);
+  }
+
+  @Test
+  public void testJsonDownload() throws Exception {
+    new JsonDownloader(downloader, new URL("http://downloads.arduino.cc/libraries/library_index.json")).download(tempFile, new MultiStepProgress(1), "");
+
+    InputStream indexIn = new FileInputStream(tempFile);
+    ObjectMapper mapper = new ObjectMapper();
+    mapper.registerModule(new MrBeanModule());
+    mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true);
+    mapper.configure(DeserializationFeature.EAGER_DESERIALIZER_FETCH, true);
+    mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+    LibrariesIndex librariesIndex = mapper.readValue(indexIn, LibrariesIndex.class);
+
+    assertTrue(librariesIndex != null);
+  }
+}

arduino-core/src/cc/arduino/contributions/GZippedJsonDownloader.java

@@ -0,0 +1,50 @@
+package cc.arduino.contributions;
+
+import cc.arduino.utils.Progress;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipUtils;
+import org.apache.commons.compress.utils.IOUtils;
+
+import java.io.*;
+import java.net.URL;
+
+public class GZippedJsonDownloader {
+
+  private final DownloadableContributionsDownloader downloader;
+  private final URL url;
+  private final URL gzippedUrl;
+
+  public GZippedJsonDownloader(DownloadableContributionsDownloader downloader, URL url, URL gzippedUrl) {
+    this.downloader = downloader;
+    this.url = url;
+    this.gzippedUrl = gzippedUrl;
+  }
+
+  public void download(File tmpFile, Progress progress, String statusText) throws Exception {
+    try {
+      new JsonDownloader(downloader, gzippedUrl).download(tmpFile, progress, statusText);
+      File gzipTmpFile = new File(tmpFile.getParentFile(), GzipUtils.getCompressedFilename(tmpFile.getName()));
+      tmpFile.renameTo(gzipTmpFile);
+      decompress(gzipTmpFile, tmpFile);
+    } catch (Exception e) {
+      new JsonDownloader(downloader, url).download(tmpFile, progress, statusText);
+    }
+  }
+
+  private void decompress(File gzipTmpFile, File tmpFile) throws IOException {
+    OutputStream os = null;
+    GzipCompressorInputStream gzipIs = null;
+    try {
+      os = new FileOutputStream(tmpFile);
+      gzipIs = new GzipCompressorInputStream(new FileInputStream(gzipTmpFile));
+      final byte[] buffer = new byte[4096];
+      int n = 0;
+      while (-1 != (n = gzipIs.read(buffer))) {
+        os.write(buffer, 0, n);
+      }
+    } finally {
+      IOUtils.closeQuietly(os);
+      IOUtils.closeQuietly(gzipIs);
+    }
+  }
+}

arduino-core/src/cc/arduino/contributions/JsonDownloader.java

@@ -0,0 +1,26 @@
+package cc.arduino.contributions;
+
+import cc.arduino.utils.Progress;
+
+import java.io.File;
+import java.net.URL;
+
+public class JsonDownloader {
+
+  private final DownloadableContributionsDownloader downloader;
+  private final URL url;
+
+  public JsonDownloader(DownloadableContributionsDownloader downloader, URL url) {
+    this.downloader = downloader;
+    this.url = url;
+  }
+
+  public void download(File tmpFile, Progress progress, String statusText) throws Exception {
+    try {
+      downloader.download(url, tmpFile, progress, statusText);
+    } catch (InterruptedException e) {
+      // Download interrupted... just exit
+      return;
+    }
+  }
+}

arduino-core/src/cc/arduino/contributions/libraries/LibraryInstaller.java

@@ -29,6 +29,7 @@
 package cc.arduino.contributions.libraries;
 
 import cc.arduino.contributions.DownloadableContributionsDownloader;
+import cc.arduino.contributions.GZippedJsonDownloader;
 import cc.arduino.utils.ArchiveExtractor;
 import cc.arduino.utils.MultiStepProgress;
 import cc.arduino.utils.Progress;
@@ -45,6 +46,7 @@ import static processing.app.I18n._;
 public class LibraryInstaller {
 
   private static final String LIBRARY_INDEX_URL;
+  private static final String LIBRARY_INDEX_URL_GZ;
 
   static {
     String externalLibraryIndexUrl = System.getProperty("LIBRARY_INDEX_URL");
@@ -53,6 +55,7 @@ public class LibraryInstaller {
     } else {
       LIBRARY_INDEX_URL = "http://downloads.arduino.cc/libraries/library_index.json";
     }
+    LIBRARY_INDEX_URL_GZ = "http://downloads.arduino.cc/libraries/library_index.json.gz";
   }
 
   private final LibrariesIndexer indexer;
@@ -77,8 +80,8 @@ public class LibraryInstaller {
     File outputFile = indexer.getIndexFile();
     File tmpFile = new File(outputFile.getAbsolutePath() + ".tmp");
     try {
-      downloader.download(url, tmpFile, progress,
+      GZippedJsonDownloader gZippedJsonDownloader = new GZippedJsonDownloader(downloader, new URL(LIBRARY_INDEX_URL), new URL(LIBRARY_INDEX_URL_GZ));
-              _("Downloading libraries index..."));
+      gZippedJsonDownloader.download(tmpFile, progress, _("Downloading libraries index..."));
     } catch (InterruptedException e) {
       // Download interrupted... just exit
       return;
@@ -91,8 +94,7 @@ public class LibraryInstaller {
     if (outputFile.exists())
       outputFile.delete();
     if (!tmpFile.renameTo(outputFile))
-      throw new Exception(
+      throw new Exception(_("An error occurred while updating libraries index!"));
-              _("An error occurred while updating libraries index!"));
 
     // Step 2: Rescan index
     rescanLibraryIndex(progress);

build/shared/revisions.txt

@@ -10,6 +10,7 @@ ARDUINO 1.6.5
 * Fixed a bug that made the IDE notify users of invalid libraries too many times. Thanks @Chris--A
 * Removed JNA. Less native stuff and less chances of incurring into an UnsatisfiedLinkError
 * Many new and old issues closed. Thanks to many, and @Chris--A in particular
+* Faster libraries list update
 
 [libraries]
 * LiquidCrystal fixes. Thanks @newbie15