第三周作业提交，单个线程的内容一次读取到内存，connection的read函数，真的无解

group24/120509419/JUnitTest/download/FileDownloaderTest.java

@@ -0,0 +1,97 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package javaclass.download;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import static java.lang.System.in;
+import java.math.BigInteger;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import javaclass.download.api.ConnectionManager;
+import javaclass.download.api.DownloadListener;
+import javaclass.download.impl.ConnectionManagerImpl;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ *
+ * @author CJ
+ */
+public class FileDownloaderTest {
+
+    boolean downloadFinished = false;
+
+    @Before
+    public void setUp() throws Exception {
+    }
+
+    @After
+    public void tearDown() throws Exception {
+    }
+
+    @Test
+    public void testDownload() throws IOException, NoSuchAlgorithmException {
+
+        String url = "http://pineappledb.xyz/test.txt";
+
+        FileDownloader downloader = new FileDownloader(url);
+
+        ConnectionManager cm = new ConnectionManagerImpl();
+        downloader.setConnectionManager(cm);
+
+        downloader.setListener(new DownloadListener() {
+            @Override
+            public void notifyFinished() {
+                downloadFinished = true;
+            }
+
+        });
+
+        downloader.setOutputFile(new File("C:\\Users\\CJ\\Desktop\\test000.txt"));
+        downloader.execute();
+
+        // 等待多线程下载程序执行完毕
+        while (!downloadFinished) {
+            try {
+                System.out.println("还没有下载完成，休眠五秒");
+                //休眠5秒
+                Thread.sleep(5000);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        }
+        System.out.println("下载完成！");
+
+        File downloadFile = downloader.getOutputFile();
+        File expectedFile = new File("C:\\Users\\CJ\\Desktop\\test.txt");
+
+        long downloadSize = downloadFile.length();
+        long expectedSize = expectedFile.length();
+
+        Assert.assertEquals(expectedSize, downloadSize);
+        FileInputStream downloadin = new FileInputStream(downloadFile);
+        FileInputStream expectedin = new FileInputStream(expectedFile);
+
+
+        MessageDigest md5 = MessageDigest.getInstance("MD5");
+        md5.update(downloadin.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, downloadFile.length()));
+        BigInteger downloadbi = new BigInteger(1, md5.digest());
+        md5.update(expectedin.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, expectedFile.length()));
+        BigInteger expectedbi = new BigInteger(1,md5.digest());
+        Assert.assertEquals(downloadbi, expectedbi);
+
+    }
+
+}

group24/120509419/javaclass/download/DownloadThread.java

@@ -0,0 +1,70 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package javaclass.download;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javaclass.download.api.Connection;
+
+public class DownloadThread extends Thread {
+
+    Connection conn;
+    int startPos;
+    int endPos;
+
+    int curStepByteSize = 1024*8;
+//    byte[] stepByteArr;
+
+    public File getCurTmpFile() {
+        return curTmpFile;
+    }
+    File curTmpFile;
+
+    public DownloadThread(Connection conn, int startPos, int endPos) {
+
+//        stepByteArr = new byte[curStepByteSize];
+
+        this.conn = conn;
+        this.startPos = startPos;
+        this.endPos = endPos;
+//        curTmpFile =
+    }
+
+    @Override
+    public void run() {
+
+        try {
+            curTmpFile = File.createTempFile("TBtools", "downloadTmp");
+//            BufferedWriter bw = new BufferedWriter(new FileWriter(curTmpFile));
+            FileOutputStream fos = new FileOutputStream(curTmpFile);
+
+
+            // 一个线程 内 分批次下载，这样才不会挤爆内存
+//            int size = endPos - startPos+1;
+//            int stepCount = size / curStepByteSize;
+////            ssint remainSize = size % curStepByteSize;
+//            for (int i = 0; i < stepCount-1; i++) {
+//                fos.write(conn.read(startPos+curStepByteSize*i, startPos+curStepByteSize*(i+1)-1));
+//                System.err.printf("Start Pos: %d\tEnd Pos %d\n",startPos+curStepByteSize*i, startPos+curStepByteSize*(i+1)-1);
+//            }
+
+//            fos.write(conn.read(startPos+curStepByteSize*(stepCount-1),endPos));
+//            System.err.printf("Start Pos: %d\tEnd Pos %d\n",startPos+curStepByteSize*(stepCount-1), endPos);
+
+            fos.write(conn.read(startPos, endPos));
+
+
+            fos.close();
+        } catch (IOException ex) {
+            Logger.getLogger(DownloadThread.class.getName()).log(Level.SEVERE, null, ex);
+        }
+    }
+}

group24/120509419/javaclass/download/FileDownloader.java

@@ -0,0 +1,129 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package javaclass.download;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javaclass.download.api.Connection;
+import javaclass.download.api.ConnectionException;
+import javaclass.download.api.ConnectionManager;
+import javaclass.download.api.DownloadListener;
+
+public class FileDownloader {
+
+    String url;
+
+    DownloadListener listener;
+
+    ConnectionManager cm;
+
+    private int numOfThreads = 2;
+
+    public File getOutputFile() {
+        return outputFile;
+    }
+
+    public void setOutputFile(File outputFile) {
+        this.outputFile = outputFile;
+    }
+
+    private File outputFile;
+
+
+    public FileDownloader(String _url) {
+        this.url = _url;
+
+
+    }
+
+    public void execute() throws IOException {
+        // 在这里实现你的代码， 注意： 需要用多线程实现下载
+        // 这个类依赖于其他几个接口, 你需要写这几个接口的实现代码
+        // (1) ConnectionManager , 可以打开一个连接，通过Connection可以读取其中的一段（用startPos, endPos来指定）
+        // (2) DownloadListener, 由于是多线程下载， 调用这个类的客户端不知道什么时候结束，所以你需要实现当所有
+        //     线程都执行完以后， 调用listener的notifiedFinished方法， 这样客户端就能收到通知。
+        // 具体的实现思路：
+        // 1. 需要调用ConnectionManager的open方法打开连接， 然后通过Connection.getContentLength方法获得文件的长度
+        // 2. 至少启动3个线程下载，  注意每个线程需要先调用ConnectionManager的open方法
+        // 然后调用read方法， read方法中有读取文件的开始位置和结束位置的参数， 返回值是byte[]数组
+        // 3. 把byte数组写入到文件中
+        // 4. 所有的线程都下载完成以后， 需要调用listener的notifiedFinished方法
+
+        // 下面的代码是示例代码， 也就是说只有一个线程， 你需要改造成多线程的。
+        Connection conn = null;
+        try {
+            // 针对每个线程，重新打开一个连接
+            conn = cm.open(this.url);
+
+            int length = conn.getContentLength();
+            System.err.println("Total Length:"+length);
+            int threadSize = length / numOfThreads;
+//            int remainSize = length % numOfThreads;
+            int threadCount = 0;
+            ExecutorService fixedThreadPool = Executors.newFixedThreadPool(numOfThreads);
+            DownloadThread[] dtArr = new DownloadThread[numOfThreads];
+            for (int i = 0; i < numOfThreads-1; i++) {
+                conn = cm.open(this.url);
+                System.err.println("Thread "+threadCount++);
+                dtArr[i] = new DownloadThread(conn, i * threadSize, (i + 1) * threadSize - 1);
+                fixedThreadPool.submit(dtArr[i]);
+            }
+            conn = cm.open(this.url);
+            dtArr[numOfThreads-1] = new DownloadThread(conn,(numOfThreads-1) * threadSize, length - 1);
+            fixedThreadPool.submit(dtArr[numOfThreads-1]);
+            fixedThreadPool.shutdown();
+            // 提交之后，等到 365天 
+            fixedThreadPool.awaitTermination(365, TimeUnit.DAYS);
+
+            // 合并所有文件
+            FileOutputStream fos = new FileOutputStream(outputFile);
+            for(DownloadThread curDt:dtArr){
+                FileInputStream fis = new FileInputStream(curDt.getCurTmpFile());
+                byte[] bufferedByteArr = new byte[1024*8];
+                int readSize;
+                while((readSize=fis.read(bufferedByteArr))!=-1){
+                    fos.write(bufferedByteArr, 0, readSize);
+                }
+                fis.close();
+            }
+            fos.close();
+
+            listener.notifyFinished();
+        } catch (ConnectionException e) {
+            e.printStackTrace();
+        } catch (InterruptedException ex) {
+            Logger.getLogger(FileDownloader.class.getName()).log(Level.SEVERE, null, ex);
+        } catch (FileNotFoundException ex) {
+            Logger.getLogger(FileDownloader.class.getName()).log(Level.SEVERE, null, ex);
+        } finally {
+            if (conn != null) {
+                conn.close();
+            }
+        }
+
+    }
+
+    public void setListener(DownloadListener listener) {
+        this.listener = listener;
+    }
+
+    public void setConnectionManager(ConnectionManager ucm) {
+        this.cm = ucm;
+    }
+
+    public DownloadListener getListener() {
+        return this.listener;
+    }
+
+}

group24/120509419/javaclass/download/api/Connection.java

@@ -0,0 +1,28 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package javaclass.download.api;
+
+import java.io.IOException;
+
+public interface Connection {
+	/**
+	 * 给定开始和结束位置， 读取数据， 返回值是字节数组
+	 * @param startPos 开始位置， 从0开始
+	 * @param endPos 结束位置
+	 * @return
+	 */
+	public byte[] read(int startPos,int endPos) throws IOException;
+	/**
+	 * 得到数据内容的长度
+	 * @return
+	 */
+	public int getContentLength();
+
+	/**
+	 * 关闭连接
+	 */
+	public void close();
+}

group24/120509419/javaclass/download/api/ConnectionException.java

@@ -0,0 +1,14 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package javaclass.download.api;
+
+/**
+ *
+ * @author CJ
+ */
+public class ConnectionException extends Exception {
+
+}

group24/120509419/javaclass/download/api/ConnectionManager.java

@@ -0,0 +1,22 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package javaclass.download.api;
+
+/**
+ *
+ * @author CJ
+ */
+public interface ConnectionManager {
+
+    /**
+     * 给定一个url , 打开一个连接
+     *
+     * @param url
+     * @return
+     * @throws javaclass.download.api.ConnectionException
+     */
+    public Connection open(String url) throws ConnectionException;
+}

group24/120509419/javaclass/download/api/DownloadListener.java

@@ -0,0 +1,14 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package javaclass.download.api;
+
+/**
+ *
+ * @author CJ
+ */
+public interface DownloadListener {
+	public void notifyFinished();
+}