Faster implementation of scp copy of www folder contents

app/src/cc/arduino/packages/uploaders/SSHUploader.java

@@ -1,17 +1,21 @@
 package cc.arduino.packages.uploaders;
 
 import cc.arduino.packages.Uploader;
-import com.jcraft.jsch.*;
+import cc.arduino.packages.uploaders.ssh.SCP;
+import cc.arduino.packages.uploaders.ssh.SSH;
+import com.jcraft.jsch.JSch;
+import com.jcraft.jsch.JSchException;
+import com.jcraft.jsch.Session;
 import processing.app.Base;
 import processing.app.Constants;
 import processing.app.NetworkMonitor;
 import processing.app.Preferences;
 import processing.app.debug.RunnerException;
 import processing.app.debug.TargetPlatform;
-import processing.app.helpers.FileUtils;
 import processing.app.helpers.PreferencesMap;
 
-import java.io.*;
+import java.io.File;
+import java.io.IOException;
 import java.util.regex.Matcher;
 
 import static processing.app.I18n._;
@@ -44,6 +48,7 @@ public class SSHUploader extends Uploader {
     }
 
     Session session = null;
+    SCP scp = null;
     try {
       JSch jSch = new JSch();
       session = jSch.getSession("root", ipAddress, 22);
@@ -52,24 +57,12 @@ public class SSHUploader extends Uploader {
       session.setUserInfo(new NetworkMonitor.NoInteractionUserInfo());
       session.connect(30000);
 
-      SCP scp = new SCP(session);
+      scp = new SCP(session);
       SSH ssh = new SSH(session);
 
-      File www = new File(sourcePath, "www");
+      scpFiles(scp, ssh, sourcePath, buildPath, className, session);
-      if (www.exists() && www.isDirectory() && www.canExecute() && canUploadWebFiles(ssh)) {
-        File destination = new File("/www/sd/" + www.getParentFile().getName());
-        ssh.execSyncCommand("mkdir -p " + FileUtils.getLinuxPathFrom(destination), System.out);
-        copyWebFiles(scp, ssh, www, destination);
-      }
 
-      String uploadedSketchFile = scp.scpHexToBoard(buildPath, className);
+      return runAVRDude(ssh);
-
-      TargetPlatform targetPlatform = Base.getTargetPlatform();
-      PreferencesMap prefs = Preferences.getMap();
-      prefs.putAll(Base.getBoardPreferences());
-      prefs.putAll(targetPlatform.getTool(prefs.get("upload.tool")));
-
-      return new SSHAVRDude(session).runAVRDude(uploadedSketchFile, verbose ? prefs.get("upload.params.verbose") : prefs.get("upload.params.quiet"));
     } catch (JSchException e) {
       if ("Auth cancel".equals(e.getMessage())) {
         return false;
@@ -78,34 +71,85 @@ public class SSHUploader extends Uploader {
     } catch (Exception e) {
       throw new RunnerException(e);
     } finally {
+      if (scp != null) {
+        try {
+          scp.close();
+        } catch (IOException e) {
+          throw new RunnerException(e);
+        }
+      }
       if (session != null) {
         session.disconnect();
       }
     }
   }
 
-  private void copyWebFiles(SCP scp, SSH ssh, File from, File destination) throws IOException, JSchException {
+  private boolean runAVRDude(SSH ssh) throws IOException, JSchException {
-    File[] files = from.listFiles();
+    TargetPlatform targetPlatform = Base.getTargetPlatform();
-    if (files == null) {
+    PreferencesMap prefs = Preferences.getMap();
-      throw new IOException("Cannot list files in " + from);
+    prefs.putAll(Base.getBoardPreferences());
-    }
+    prefs.putAll(targetPlatform.getTool(prefs.get("upload.tool")));
-    for (File file : files) {
+
-      if (file.isDirectory() && file.canExecute()) {
+    String additionalParams = verbose ? prefs.get("upload.params.verbose") : prefs.get("upload.params.quiet");
-        File newDestination = new File(destination, file.getName());
+
-        ssh.execSyncCommand("mkdir -p " + FileUtils.getLinuxPathFrom(newDestination), System.out);
+    boolean success = ssh.execSyncCommand("merge-sketch-with-bootloader /tmp/sketch.hex", System.out, System.err);
-        copyWebFiles(scp, ssh, file, newDestination);
+    ssh.execSyncCommand("kill-bridge");
-      } else {
+    success = success && ssh.execSyncCommand("run-avrdude /tmp/sketch.hex '" + additionalParams + "'", System.out, System.err);
-        scp.scpFile(file, FileUtils.getLinuxPathFrom(new File(destination, file.getName())));
+    return success;
+  }
+
+  private void scpFiles(SCP scp, SSH ssh, File sourcePath, String buildPath, String className, Session session) throws JSchException, IOException {
+    try {
+      scp.open();
+      scp.startFolder("tmp");
+      scp.sendFile(new File(buildPath, className + ".hex"), "sketch.hex");
+      scp.endFolder();
+
+      if (canUploadWWWFiles(sourcePath, ssh)) {
+        scp.startFolder("www");
+        scp.startFolder("sd");
+        scp.startFolder(sourcePath.getName());
+        recursiveSCP(new File(sourcePath, "www"), scp);
+        scp.endFolder();
+        scp.endFolder();
+        scp.endFolder();
       }
+    } finally {
+      scp.close();
     }
   }
 
-  private boolean canUploadWebFiles(SSH ssh) throws JSchException, IOException {
+  private boolean canUploadWWWFiles(File sourcePath, SSH ssh) throws IOException, JSchException {
-    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    File www = new File(sourcePath, "www");
-    PrintStream ps = new PrintStream(baos);
+    if (!www.exists() || !www.isDirectory()) {
-    ssh.execSyncCommand("if [ -L /www/sd ] && [ -d /www/sd ]; then echo 1; else echo 0; fi", ps);
+      return false;
-    String output = new String(baos.toByteArray());
+    }
-    return "1".equals(output.trim());
+    if (!www.canExecute()) {
+      System.out.println("Problem accessing files in folder " + www);
+      return false;
+    }
+    if (!ssh.execSyncCommand("special-storage-available")) {
+      System.out.println("Problem accessing board folder /www/sd");
+      return false;
+    }
+    return true;
+  }
+
+  private void recursiveSCP(File from, SCP scp) throws IOException {
+    File[] files = from.listFiles();
+    if (files == null) {
+      return;
+    }
+
+    for (File file : files) {
+      if (file.isDirectory() && file.canExecute()) {
+        scp.startFolder(file.getName());
+        recursiveSCP(file, scp);
+        scp.endFolder();
+      } else if (file.isFile() && file.canRead()) {
+        scp.sendFile(file);
+      }
+    }
   }
 
   @Override
@@ -113,193 +157,4 @@ public class SSHUploader extends Uploader {
     throw new RunnerException("Can't burn bootloader via SSH");
   }
 
-  private static class SSH {
-
-    protected final Session session;
-
-    public SSH(Session session) {
-      this.session = session;
-    }
-
-    protected boolean execSyncCommand(String command, PrintStream stdoutConsumer) throws JSchException, IOException {
-      return execSyncCommand(command, stdoutConsumer, null);
-    }
-
-    protected boolean execSyncCommand(String command, PrintStream stdoutConsumer, PrintStream stderrConsumer) throws JSchException, IOException {
-      InputStream stdout = null;
-      InputStream stderr = null;
-      Channel channel = null;
-      try {
-        channel = session.openChannel("exec");
-        ((ChannelExec) channel).setCommand(command);
-
-        channel.setInputStream(null);
-
-        stdout = channel.getInputStream();
-        if (stderrConsumer != null) {
-          stderr = ((ChannelExec) channel).getErrStream();
-        }
-
-        channel.connect();
-
-        int exitCode = consumeOutputSyncAndReturnExitCode(channel, stdout, stdoutConsumer, stderr, stderrConsumer);
-
-        return stderrConsumer == null || exitCode == 0;
-
-      } finally {
-        if (stdout != null) {
-          stdout.close();
-        }
-        if (stderr != null) {
-          stderr.close();
-        }
-        if (channel != null) {
-          channel.disconnect();
-        }
-      }
-    }
-
-    protected int consumeOutputSyncAndReturnExitCode(Channel channel, InputStream stdout, PrintStream stdoutConsumer, InputStream stderr, PrintStream stderrConsumer) throws IOException {
-      byte[] tmp = new byte[102400];
-      while (true) {
-        consumeStream(tmp, stdout, stdoutConsumer);
-        consumeStream(tmp, stderr, stderrConsumer);
-
-        if (channel.isClosed()) {
-          return channel.getExitStatus();
-        }
-        try {
-          Thread.sleep(100);
-        } catch (Exception ee) {
-          // noop
-        }
-      }
-    }
-
-    private void consumeStream(byte[] buffer, InputStream in, PrintStream out) throws IOException {
-      while (in != null && in.available() > 0) {
-        int length = in.read(buffer, 0, buffer.length);
-        if (length < 0) {
-          break;
-        }
-        out.print(new String(buffer, 0, length));
-      }
-    }
-
-  }
-
-  private static class SSHAVRDude extends SSH {
-
-    public SSHAVRDude(Session session) {
-      super(session);
-    }
-
-    public boolean runAVRDude(String sketchFile, String additionalParams) throws IOException, JSchException {
-      boolean success = execSyncCommand("merge-sketch-with-bootloader " + sketchFile, System.out, System.err);
-      success = success && execSyncCommand("kill-bridge", System.out);
-      success = success && execSyncCommand("run-avrdude " + sketchFile + " '" + additionalParams + "'", System.out, System.err);
-      return success;
-    }
-
-  }
-
-  private static class SCP extends SSH {
-
-    private static final String SKETCH_FILE = "/tmp/sketch.hex";
-
-    public SCP(Session session) {
-      super(session);
-    }
-
-    public void scpFile(File from, String absolutePathToDestination) throws JSchException, IOException {
-      Channel channel = null;
-      OutputStream out = null;
-      InputStream in = null;
-      try {
-        channel = session.openChannel("exec");
-        ((ChannelExec) channel).setCommand("scp -t " + absolutePathToDestination);
-
-        out = channel.getOutputStream();
-        in = channel.getInputStream();
-
-        channel.connect();
-
-        ensureAcknowledged(out, in);
-
-        sendFileSizeAndName(out, in, from);
-        ensureAcknowledged(out, in);
-
-        sendFileContents(out, from);
-        ensureAcknowledged(out, in);
-      } finally {
-        if (out != null) {
-          out.close();
-        }
-        if (in != null) {
-          in.close();
-        }
-        if (channel != null) {
-          channel.disconnect();
-        }
-      }
-    }
-
-    public String scpHexToBoard(String buildPath, String className) throws JSchException, IOException {
-      scpFile(new File(buildPath, className + ".hex"), SKETCH_FILE);
-      return SKETCH_FILE;
-    }
-
-    private void ensureAcknowledged(OutputStream out, InputStream in) throws IOException {
-      out.flush();
-
-      int b = in.read();
-
-      if (b == 0) return;
-      if (b == -1) return;
-
-      if (b == 1 || b == 2) {
-        StringBuilder sb = new StringBuilder();
-        sb.append("SCP error: ");
-
-        int c;
-        do {
-          c = in.read();
-          sb.append((char) c);
-        } while (c != '\n');
-
-        throw new IOException(sb.toString());
-      }
-
-      throw new IOException("Uknown SCP error: " + b);
-    }
-
-    private void sendFileContents(OutputStream out, File hex) throws IOException {
-      FileInputStream fis = null;
-      try {
-        fis = new FileInputStream(hex);
-        byte[] buf = new byte[4096];
-        while (true) {
-          int len = fis.read(buf, 0, buf.length);
-          if (len <= 0) break;
-          out.write(buf, 0, len);
-        }
-
-        // \0 terminates file
-        buf[0] = 0;
-        out.write(buf, 0, 1);
-      } finally {
-        if (fis != null) {
-          fis.close();
-        }
-      }
-    }
-
-    private void sendFileSizeAndName(OutputStream out, InputStream in, File hex) throws IOException {
-      long filesize = hex.length();
-      String command = "C0644 " + filesize + " " + hex.getName() + "\n";
-      out.write(command.getBytes());
-    }
-
-  }
-
 }

app/src/cc/arduino/packages/uploaders/ssh/SCP.java

@@ -0,0 +1,111 @@
+package cc.arduino.packages.uploaders.ssh;
+
+import com.jcraft.jsch.Channel;
+import com.jcraft.jsch.ChannelExec;
+import com.jcraft.jsch.JSchException;
+import com.jcraft.jsch.Session;
+
+import java.io.*;
+
+public class SCP extends SSH {
+
+  private Channel channel;
+  private OutputStream out;
+  private InputStream in;
+
+  public SCP(Session session) {
+    super(session);
+  }
+
+  public void open() throws JSchException, IOException {
+    try {
+      channel = session.openChannel("exec");
+      ((ChannelExec) channel).setCommand("scp -t -r -d /");
+
+      out = channel.getOutputStream();
+      in = channel.getInputStream();
+
+      channel.connect();
+      ensureAcknowledged();
+    } catch (Exception e) {
+      close();
+    }
+  }
+
+  public void close() throws IOException {
+    if (out != null) {
+      out.close();
+    }
+    if (in != null) {
+      in.close();
+    }
+    if (channel != null) {
+      channel.disconnect();
+    }
+  }
+
+  protected void ensureAcknowledged() throws IOException {
+    out.flush();
+
+    int b = in.read();
+
+    if (b == 0) return;
+    if (b == -1) return;
+
+    if (b == 1 || b == 2) {
+      StringBuilder sb = new StringBuilder();
+      sb.append("SCP error: ");
+
+      int c;
+      do {
+        c = in.read();
+        sb.append((char) c);
+      } while (c != '\n');
+
+      throw new IOException(sb.toString());
+    }
+
+    throw new IOException("Uknown SCP error: " + b);
+  }
+
+  public void sendFile(File localFile) throws IOException {
+    sendFile(localFile, localFile.getName());
+  }
+
+  public void sendFile(File localFile, String remoteFile) throws IOException {
+    out.write(("C0644 " + localFile.length() + " " + remoteFile + "\n").getBytes());
+    ensureAcknowledged();
+
+    FileInputStream fis = null;
+    try {
+      fis = new FileInputStream(localFile);
+      byte[] buf = new byte[4096];
+      while (true) {
+        int len = fis.read(buf, 0, buf.length);
+        if (len <= 0) break;
+        out.write(buf, 0, len);
+      }
+
+      // \0 terminates file
+      buf[0] = 0;
+      out.write(buf, 0, 1);
+    } finally {
+      if (fis != null) {
+        fis.close();
+      }
+    }
+
+    ensureAcknowledged();
+  }
+
+  public void startFolder(String folder) throws IOException {
+    out.write(("D0755 0 " + folder + "\n").getBytes());
+    ensureAcknowledged();
+  }
+
+  public void endFolder() throws IOException {
+    out.write("E\n".getBytes());
+    ensureAcknowledged();
+  }
+
+}

app/src/cc/arduino/packages/uploaders/ssh/SSH.java

@@ -0,0 +1,85 @@
+package cc.arduino.packages.uploaders.ssh;
+
+import com.jcraft.jsch.Channel;
+import com.jcraft.jsch.ChannelExec;
+import com.jcraft.jsch.JSchException;
+import com.jcraft.jsch.Session;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintStream;
+
+public class SSH {
+
+  protected final Session session;
+
+  public SSH(Session session) {
+    this.session = session;
+  }
+
+  public boolean execSyncCommand(String command) throws JSchException, IOException {
+    return execSyncCommand(command, null, null);
+  }
+
+  public boolean execSyncCommand(String command, PrintStream stdoutConsumer, PrintStream stderrConsumer) throws JSchException, IOException {
+    InputStream stdout = null;
+    InputStream stderr = null;
+    Channel channel = null;
+    try {
+      channel = session.openChannel("exec");
+      ((ChannelExec) channel).setCommand(command);
+
+      channel.setInputStream(null);
+
+      stdout = channel.getInputStream();
+      stderr = ((ChannelExec) channel).getErrStream();
+
+      channel.connect();
+
+      int exitCode = consumeOutputSyncAndReturnExitCode(channel, stdout, stdoutConsumer, stderr, stderrConsumer);
+
+      return exitCode == 0;
+
+    } finally {
+      if (stdout != null) {
+        stdout.close();
+      }
+      if (stderr != null) {
+        stderr.close();
+      }
+      if (channel != null) {
+        channel.disconnect();
+      }
+    }
+  }
+
+  protected int consumeOutputSyncAndReturnExitCode(Channel channel, InputStream stdout, PrintStream stdoutConsumer, InputStream stderr, PrintStream stderrConsumer) throws IOException {
+    byte[] tmp = new byte[102400];
+    while (true) {
+      consumeStream(tmp, stdout, stdoutConsumer);
+      consumeStream(tmp, stderr, stderrConsumer);
+
+      if (channel.isClosed()) {
+        return channel.getExitStatus();
+      }
+      try {
+        Thread.sleep(100);
+      } catch (Exception ee) {
+        // noop
+      }
+    }
+  }
+
+  protected void consumeStream(byte[] buffer, InputStream in, PrintStream out) throws IOException {
+    while (in.available() > 0) {
+      int length = in.read(buffer, 0, buffer.length);
+      if (length < 0) {
+        break;
+      }
+      if (out != null) {
+        out.print(new String(buffer, 0, length));
+      }
+    }
+  }
+
+}