Download a list a webpages into your computer

Of course you know that we won't stop at copying one file. I want to copy some lecture notes at http://www.charlesli.org/java/articles/copyURL/test/note1.txt .... http://www.charlesli.org/java/articles/copyURL/test/note4.txt. to test/note1.txt .. etc

Example 3

import java.net.*;
import java.io.*;

public class CopyURL3 {
    // copy url to local file name
    public static void copy(String urlname, String filename) throws Exception {
        URL url = new URL(urlname);
        File file = new File(filename);
        BufferedInputStream in = new BufferedInputStream(url.openConnection().getInputStream());
        FileOutputStream out = new FileOutputStream(file);
        int ch = 0;
        while((ch = in.read())!=-1) {
            out.write(ch);
        }
        
    }
    
    public static void main(String[] args) throws Exception {
        String urlname = null;
        String filename = null;
        String dirname = "test";
        // create directories. make sure the directories exist
        new File(dirname).mkdirs();
        
        // use for timing, can be deleted
        long startTime;
        long endTime; // for measuring time
        double time; // time in second
        
        for(int i = 1; i <= 4; i++) {
            urlname = "http://www.charlesli.org/java/articles/copyURL/test/note" + i + ".txt";
            filename = dirname + "/note" + i + ".txt";
            System.out.println("Copying " + urlname + " to " + filename + "\n");
            try {
                startTime = System.currentTimeMillis();// for timing
                copy(urlname, filename);
                endTime = System.currentTimeMillis(); // for timing
                time = (endTime - startTime)/1000.; // for timing
                System.out.println("File size: " + new File(filename).length());
                System.out.println("Time: " + time + "s\n"); // for timing 
            } catch (Exception ex) {
                // do something
                System.out.println("Fail to copy\n" + urlname + "\n");
                ex.printStackTrace();
            }
        }
    }
}

Explanation

• download 4 webpages into a folder named test
• mkdirs() creates all the necessary directories for creating the files.
• catch exception in copy(urlname, filename) to avoid termination of the program. The try-catch ensures the program goes on.
• ex.printStackTrace() is the standard exception error message. Line numbers and the error types will be displayed.
• Use System.currentTimeMillis() for timing.

Example 4

import java.net.*;
import java.io.*;

public class CopyURL4 {
    // copy url to local file name
    public static void copy(String urlname, String filename) throws Exception {
        URL url = new URL(urlname);
        BufferedInputStream in = new BufferedInputStream(url.openConnection().getInputStream());
        FileOutputStream out = new FileOutputStream(filename);
        int ch = 0;
        while((ch = in.read())!=-1) {
            out.write(ch);
        }
        in.close();
        out.close();
        
    }
    
    // copy file src to file target
    public static void copyFile(String src, String target) throws Exception {
        BufferedInputStream in = new BufferedInputStream(new FileInputStream(src));
        FileOutputStream out = new FileOutputStream(target);
        int ch = 0;
        while((ch = in.read())!=-1) {
            out.write(ch);
        }
        in.close();
        out.close();
        
    }
    
    // back up file by adding .bak to the original file
    // if .bak exits, add .bak2, etc etc
    public static void backUp(String filename) throws Exception{
        String backUpFileName = filename + ".bak";
        int i = 1;
        while(new File(backUpFileName).exists()) {
            i++;
            backUpFileName = filename + ".bak" + i;
        }
       
        copyFile(filename, backUpFileName);

    }
    
    public static void main(String[] args) throws Exception {        
        String urlname = null;
        String filename = null;
        File targetFile = null; 
        File backUpFile = null; 
        
        String urlHead = "http://www.charlesli.org/java/articles/copyURL/test/note";
        String urlTail = ".txt";
        String fileHead = "test/note";
        String fileTail = ".txt";
        
        long startTime;
        long endTime; // for measuring time
        double time; // time in second
        
        for(int i = 1; i <= 4; i++) {
           urlname = urlHead + i + urlTail;
           filename = fileHead + i + fileTail;
           targetFile = new File(filename);
           if(targetFile.exists()) {
               System.out.println("File " + targetFile + " exists.");
               // what do you want to do? 
               // I made a back up file by adding .bak at the end
               try {
                   System.out.println("Backing up the file.");
                   backUp(filename);
               } catch (Exception ex) {
                   // fail in backing out.
                   // go the next loop 
                   System.out.println("Fail to back up the file.");
               }
           }
           
           targetFile.getParentFile().mkdirs(); // make sure that the necessary folder exits
           System.out.println("Copying " + urlname + " to " + filename);
           try {
               startTime = System.currentTimeMillis();
               copy(urlname, filename);
               endTime = System.currentTimeMillis();
               time = (endTime - startTime)/1000.;
               System.out.println("File size: " + new File(filename).length());
               System.out.println("Time: " + time + "s\n");
               
           } catch (Exception ex) {
               // do something
               System.out.println("Fail to copy\n" + urlname + "\n");
               ex.printStackTrace();
           }
        }
    }
}